From c65a16ce8fcc79a4e0ae69a7bab6f5ff9c40a61d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 13 Oct 2026 14:37:51 +0200 Subject: [PATCH 01/33] add shared crate with shared constants and client security enum --- Cargo.lock | 8 ++++++++ Cargo.nix | 34 +++++++++++++++++++++++++++++++++ Cargo.toml | 3 ++- rust/shared/Cargo.toml | 15 +++++++++++++++ rust/shared/src/agent.rs | 41 ++++++++++++++++++++++++++++++++++++++++ rust/shared/src/lib.rs | 1 + 6 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 rust/shared/Cargo.toml create mode 100644 rust/shared/src/agent.rs create mode 100644 rust/shared/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index be730927..a8762886 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3285,6 +3285,14 @@ dependencies = [ "tracing", ] +[[package]] +name = "stackable-kafka-shared" +version = "0.0.0-dev" +dependencies = [ + "rstest", + "strum", +] + [[package]] name = "stackable-operator" version = "0.119.0" diff --git a/Cargo.nix b/Cargo.nix index 7003815a..480c24ee 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -69,6 +69,16 @@ rec { packageId = "stackable-kafka-operator"; }; + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; + "stackable-kafka-shared" = rec { + packageId = "stackable-kafka-shared"; + build = internal.buildRustCrateWithFeatures { + packageId = "stackable-kafka-shared"; + }; + # Debug support which might change between releases. # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; @@ -10803,6 +10813,30 @@ rec { } ]; + }; + "stackable-kafka-shared" = rec { + crateName = "stackable-kafka-shared"; + version = "0.0.0-dev"; + edition = "2024"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./rust/shared; }; + libName = "stackable_kafka_shared"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "strum"; + packageId = "strum"; + features = [ "derive" ]; + } + ]; + devDependencies = [ + { + name = "rstest"; + packageId = "rstest"; + } + ]; + }; "stackable-operator" = rec { crateName = "stackable-operator"; diff --git a/Cargo.toml b/Cargo.toml index 540a2aeb..423ccc60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rust/operator-binary", "rust/agent-binary", "rust/kafka-admin"] +members = ["rust/operator-binary", "rust/agent-binary", "rust/kafka-admin", "rust/shared"] resolver = "2" [workspace.package] @@ -11,6 +11,7 @@ repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["crds", "webhook"] } +stackable-kafka-shared = { path = "rust/shared" } anyhow = "1.0" base64 = "0.22" diff --git a/rust/shared/Cargo.toml b/rust/shared/Cargo.toml new file mode 100644 index 00000000..4164d809 --- /dev/null +++ b/rust/shared/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "stackable-kafka-shared" +description = "Types and contracts shared by the Stackable operator and agent for Apache Kafka" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true +publish = false + +[dependencies] +strum.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/rust/shared/src/agent.rs b/rust/shared/src/agent.rs new file mode 100644 index 00000000..4af72434 --- /dev/null +++ b/rust/shared/src/agent.rs @@ -0,0 +1,41 @@ +//! Paths and environment variables through which the operator configures the agent. + +use strum::{Display, EnumString}; + +pub const DISCOVERY_DIR: &str = "/stackable/discovery"; + +/// Directory of the CA certificate the Kafka brokers are verified with. +pub const SERVER_CA_DIR: &str = "/stackable/secrets/tls-server-ca"; + +pub const ENV_CLUSTER_NAME: &str = "KAFKA_CLUSTER_NAME"; + +pub const ENV_CLUSTER_NAMESPACE: &str = "KAFKA_CLUSTER_NAMESPACE"; + +pub const ENV_CLIENT_SECURITY: &str = "KAFKA_CLIENT_SECURITY"; + +/// How the agent connects to the brokers. +#[derive(Clone, Copy, Debug, Display, EnumString, Eq, PartialEq)] +#[strum(serialize_all = "kebab-case")] +pub enum ClientSecurity { + Plaintext, + Tls, + MutualTls, +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::plaintext(ClientSecurity::Plaintext, "plaintext")] + #[case::tls(ClientSecurity::Tls, "tls")] + #[case::mutual_tls(ClientSecurity::MutualTls, "mutual-tls")] + fn roundtrip(#[case] security: ClientSecurity, #[case] value: &str) { + assert_eq!(value, security.to_string()); + assert_eq!(Ok(security), ClientSecurity::from_str(value)); + } +} diff --git a/rust/shared/src/lib.rs b/rust/shared/src/lib.rs new file mode 100644 index 00000000..f17bc55d --- /dev/null +++ b/rust/shared/src/lib.rs @@ -0,0 +1 @@ +pub mod agent; From 41db2c70adf507669c9a81f14d943329d79f08dd Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 13 Oct 2026 17:14:37 +0200 Subject: [PATCH 02/33] add KafkaTopic CRD --- Cargo.lock | 7 + Cargo.nix | 34 +++- Cargo.toml | 1 + deploy/helm/kafka-operator/Chart.yaml | 5 + extra/crds.yaml | 138 ++++++++++++++ rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/main.rs | 9 +- .../src/webhooks/conversion.rs | 15 +- rust/shared/Cargo.toml | 10 ++ rust/shared/src/crd/mod.rs | 1 + rust/shared/src/crd/topic.rs | 168 ++++++++++++++++++ rust/shared/src/lib.rs | 1 + 12 files changed, 383 insertions(+), 7 deletions(-) create mode 100644 rust/shared/src/crd/mod.rs create mode 100644 rust/shared/src/crd/topic.rs diff --git a/Cargo.lock b/Cargo.lock index a8762886..a50ea695 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3279,6 +3279,7 @@ dependencies = [ "serde_json", "serde_yaml", "snafu 0.9.2", + "stackable-kafka-shared", "stackable-operator", "strum", "tokio", @@ -3289,7 +3290,13 @@ dependencies = [ name = "stackable-kafka-shared" version = "0.0.0-dev" dependencies = [ + "indoc", + "regex", "rstest", + "serde", + "serde_json", + "snafu 0.9.2", + "stackable-operator", "strum", ] diff --git a/Cargo.nix b/Cargo.nix index 480c24ee..95a0e642 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10775,6 +10775,10 @@ rec { name = "snafu"; packageId = "snafu 0.9.2"; } + { + name = "stackable-kafka-shared"; + packageId = "stackable-kafka-shared"; + } { name = "stackable-operator"; packageId = "stackable-operator"; @@ -10824,6 +10828,28 @@ rec { "Stackable GmbH " ]; dependencies = [ + { + name = "regex"; + packageId = "regex"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "snafu"; + packageId = "snafu 0.9.2"; + } + { + name = "stackable-operator"; + packageId = "stackable-operator"; + features = [ "crds" "webhook" ]; + } { name = "strum"; packageId = "strum"; @@ -10831,12 +10857,18 @@ rec { } ]; devDependencies = [ + { + name = "indoc"; + packageId = "indoc"; + } { name = "rstest"; packageId = "rstest"; } ]; - + features = { + }; + resolvedDefaultFeatures = [ "test-support" ]; }; "stackable-operator" = rec { crateName = "stackable-operator"; diff --git a/Cargo.toml b/Cargo.toml index 423ccc60..dfe32d4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ const_format = "0.2" futures = "0.3" indoc = "2.0" kafka-protocol = { version = "0.18", default-features = false, features = ["broker", "client"] } +regex = "1.13" rstest = "0.26" rustls = { version = "0.23", default-features = false, features = ["logging", "ring", "std", "tls12"] } semver = "1.0" diff --git a/deploy/helm/kafka-operator/Chart.yaml b/deploy/helm/kafka-operator/Chart.yaml index aca052b2..e8472c04 100644 --- a/deploy/helm/kafka-operator/Chart.yaml +++ b/deploy/helm/kafka-operator/Chart.yaml @@ -31,3 +31,8 @@ annotations: name: kafkaclusters.kafka.stackable.tech displayName: KafkaCluster description: A Kafka cluster stacklet. This resource is managed by the Stackable operator for Apache Kafka. + - kind: KafkaTopic + version: v1alpha1 + name: kafkatopics.kafka.stackable.tech + displayName: KafkaTopic + description: A Kafka topic, created and updated by the agent of the referenced Kafka cluster. diff --git a/extra/crds.yaml b/extra/crds.yaml index 83287899..3c824878 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2236,3 +2236,141 @@ spec: storage: true subresources: status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kafkatopics.kafka.stackable.tech +spec: + group: kafka.stackable.tech + names: + kind: KafkaTopic + plural: kafkatopics + singular: kafkatopic + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: A Kafka topic, created and updated by the agent of the referenced Kafka cluster. + properties: + spec: + description: |- + A Kafka topic managed by the agent of the referenced Kafka cluster. + + The referenced `KafkaCluster` must have `platformAccess` enabled or this resource will + not be reconciled. + properties: + clusterName: + description: Name of a KafkaCluster in the namespace of this KafkaTopic resource. Immutable. + maxLength: 40 + minLength: 1 + type: string + x-kubernetes-validations: + - message: clusterName is immutable + rule: self == oldSelf + config: + additionalProperties: + type: string + default: {} + description: |- + Topic configs like `retention.ms`. Keys removed from this map are reset to the broker + default. Keys not listed here are not managed by the agent. + type: object + managementPolicy: + default: + adoption: Refuse + deletion: Retain + description: What an agent may do with a resource inside the product beyond applying its spec. + properties: + adoption: + default: Refuse + description: |- + Whether a resource that already exists in the product, but was not created by the + agent, is taken over. `Refuse` by default. + enum: + - Refuse + - Adopt + type: string + deletion: + default: Retain + description: |- + What happens to the resource in the product when this object is deleted. `Retain` by + default. + enum: + - Retain + - Delete + type: string + type: object + partitions: + description: Number of partitions. Can only be increased. + format: int32 + minimum: 1.0 + type: integer + replicationFactor: + description: Number of replicas of each partition. Changes are not applied to existing topics. + format: int16 + maximum: 32767.0 + minimum: 1.0 + type: integer + topicName: + description: Name of the topic in Kafka. Immutable. + maxLength: 249 + minLength: 1 + pattern: ^([a-zA-Z0-9-][a-zA-Z0-9._-]*|_([a-zA-Z0-9.-][a-zA-Z0-9._-]*)?|\.[a-zA-Z0-9_-][a-zA-Z0-9._-]*|\.\.[a-zA-Z0-9._-]+)$ + type: string + x-kubernetes-validations: + - message: topicName is immutable + rule: self == oldSelf + required: + - clusterName + - partitions + - replicationFactor + - topicName + type: object + status: + nullable: true + properties: + conditions: + default: [] + description: '`Ready` and `Synced`.' + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + type: string + status: + description: status of the condition, one of True, False, Unknown. + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + required: + - spec + title: KafkaTopic + type: object + served: true + storage: true + subresources: + status: {} diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 5c926bea..4adf9086 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true publish = false [dependencies] +stackable-kafka-shared.workspace = true stackable-operator.workspace = true indoc.workspace = true diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 809bd566..37020491 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use anyhow::anyhow; use clap::Parser; use futures::{FutureExt, StreamExt, TryFutureExt}; +use stackable_kafka_shared::crd::topic::{KafkaTopic, KafkaTopicVersion}; use stackable_operator::{ YamlSchema, cli::{Command, RunArguments}, @@ -77,8 +78,12 @@ struct KafkaRun { async fn main() -> anyhow::Result<()> { let opts = Opts::parse(); match opts.cmd { - Command::Crd => KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1)? - .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?, + Command::Crd => { + KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1)? + .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; + KafkaTopic::merged_crd(KafkaTopicVersion::V1Alpha1)? + .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; + } Command::Run(KafkaRun { common: RunArguments { diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index 232e4dac..7c269365 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -1,4 +1,5 @@ use snafu::{ResultExt, Snafu}; +use stackable_kafka_shared::crd::topic::{KafkaTopic, KafkaTopicVersion}; use stackable_operator::{ cli::OperatorEnvironmentOptions, kube::{Client, core::crd::MergeError}, @@ -28,10 +29,16 @@ pub async fn create_webhook_server( disable_crd_maintenance: bool, client: Client, ) -> Result { - let crds_and_handlers = vec![( - KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1).context(MergeCrdSnafu)?, - KafkaCluster::try_convert, - )]; + let crds_and_handlers = vec![ + ( + KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1).context(MergeCrdSnafu)?, + KafkaCluster::try_convert as fn(_) -> _, + ), + ( + KafkaTopic::merged_crd(KafkaTopicVersion::V1Alpha1).context(MergeCrdSnafu)?, + KafkaTopic::try_convert as fn(_) -> _, + ), + ]; let conversion_webhook_options = ConversionWebhookOptions { disable_crd_maintenance, diff --git a/rust/shared/Cargo.toml b/rust/shared/Cargo.toml index 4164d809..9eedd8c6 100644 --- a/rust/shared/Cargo.toml +++ b/rust/shared/Cargo.toml @@ -9,7 +9,17 @@ repository.workspace = true publish = false [dependencies] +stackable-operator.workspace = true + +regex.workspace = true +serde.workspace = true +serde_json.workspace = true +snafu.workspace = true strum.workspace = true [dev-dependencies] +indoc.workspace = true rstest.workspace = true + +[features] +test-support = [] diff --git a/rust/shared/src/crd/mod.rs b/rust/shared/src/crd/mod.rs new file mode 100644 index 00000000..8a7c2fb7 --- /dev/null +++ b/rust/shared/src/crd/mod.rs @@ -0,0 +1 @@ +pub mod topic; diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs new file mode 100644 index 00000000..7b855a8e --- /dev/null +++ b/rust/shared/src/crd/topic.rs @@ -0,0 +1,168 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + attributed_string_type, + crd::platform_access::v1alpha1::ManagementPolicy, + k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition, + kube::CustomResource, + schemars::{self, JsonSchema}, + v2::types::operator::ClusterName, + versioned::versioned, +}; + +attributed_string_type! { + TopicName, + "The name of a Kafka topic, not `.` or `..` and not starting with `__`", + "shop.orders", + (min_length = 1), + (max_length = 249), + (regex = r"^([a-zA-Z0-9-][a-zA-Z0-9._-]*|_([a-zA-Z0-9.-][a-zA-Z0-9._-]*)?|\.[a-zA-Z0-9_-][a-zA-Z0-9._-]*|\.\.[a-zA-Z0-9._-]+)$") +} + +#[versioned( + version(name = "v1alpha1"), + crates( + kube_core = "stackable_operator::kube::core", + kube_client = "stackable_operator::kube::client", + k8s_openapi = "stackable_operator::k8s_openapi", + schemars = "stackable_operator::schemars", + versioned = "stackable_operator::versioned" + ) +)] +pub mod versioned { + /// A Kafka topic managed by the agent of the referenced Kafka cluster. + /// + /// The referenced `KafkaCluster` must have `platformAccess` enabled or this resource will + /// not be reconciled. + #[versioned(crd( + doc = "A Kafka topic, created and updated by the agent of the referenced Kafka cluster.", + group = "kafka.stackable.tech", + plural = "kafkatopics", + status = "KafkaTopicStatus", + namespaced + ))] + #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaTopicSpec { + /// Name of a KafkaCluster in the namespace of this KafkaTopic resource. Immutable. + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "self == oldSelf", + "message": "clusterName is immutable" + }]))] + pub cluster_name: ClusterName, + + #[serde(default)] + pub management_policy: ManagementPolicy, + + /// Name of the topic in Kafka. Immutable. + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "self == oldSelf", + "message": "topicName is immutable" + }]))] + pub topic_name: TopicName, + + /// Number of partitions. Can only be increased. + #[schemars(range(min = 1))] + pub partitions: i32, + + /// Number of replicas of each partition. Changes are not applied to existing topics. + #[schemars(range(min = 1))] + pub replication_factor: i16, + + /// Topic configs like `retention.ms`. Keys removed from this map are reset to the broker + /// default. Keys not listed here are not managed by the agent. + #[serde(default)] + pub config: BTreeMap, + } + + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaTopicStatus { + /// `Ready` and `Synced`. + #[serde(default)] + pub conditions: Vec, + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use rstest::rstest; + use stackable_operator::versioned::test_utils::RoundtripTestData; + + use super::*; + + impl RoundtripTestData for v1alpha1::KafkaTopicSpec { + fn roundtrip_test_data() -> Vec { + stackable_operator::utils::yaml_from_str_singleton_map(indoc::indoc! {r#" + - clusterName: simple-kafka + managementPolicy: + adoption: Adopt + deletion: Delete + topicName: shop.orders + partitions: 6 + replicationFactor: 3 + config: + retention.ms: "604800000" + - clusterName: simple-kafka + topicName: minimal + partitions: 1 + replicationFactor: 1 + "#}) + .expect("the roundtrip test data should be valid YAML") + } + } + + #[rstest] + #[case::dotted("shop.orders")] + #[case::uppercase_and_underscore("Shop_Orders-1")] + #[case::single_underscore("_internal")] + #[case::underscore_only("_")] + #[case::leading_dot(".hidden")] + #[case::three_dots("...")] + fn accept_topic_name(#[case] name: &str) { + assert!( + TopicName::from_str(name).is_ok(), + "{name:?} should be valid" + ); + } + + #[rstest] + #[case::empty("")] + #[case::dot(".")] + #[case::two_dots("..")] + #[case::double_underscore_prefix("__consumer_offsets")] + #[case::invalid_character("shop/orders")] + #[case::too_long(&"a".repeat(250))] + fn reject_topic_name(#[case] name: &str) { + assert!( + TopicName::from_str(name).is_err(), + "{name:?} should be invalid" + ); + } + + #[test] + fn schema_contains_validations() { + let crd = serde_json::to_value( + KafkaTopic::merged_crd(KafkaTopicVersion::V1Alpha1) + .expect("the CRD should be mergeable"), + ) + .expect("the CRD should be serializable"); + let spec = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]["properties"]; + + assert_eq!( + "self == oldSelf", + spec["clusterName"]["x-kubernetes-validations"][0]["rule"] + ); + assert_eq!( + "self == oldSelf", + spec["topicName"]["x-kubernetes-validations"][0]["rule"] + ); + assert_eq!(249, spec["topicName"]["maxLength"]); + assert!(spec["topicName"]["pattern"].is_string()); + assert_eq!(Some(1.0), spec["partitions"]["minimum"].as_f64()); + assert_eq!(Some(1.0), spec["replicationFactor"]["minimum"].as_f64()); + } +} diff --git a/rust/shared/src/lib.rs b/rust/shared/src/lib.rs index f17bc55d..0e0beca0 100644 --- a/rust/shared/src/lib.rs +++ b/rust/shared/src/lib.rs @@ -1 +1,2 @@ pub mod agent; +pub mod crd; From fe5203f4075e733f95823ace251a460b79e0355a Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 14 Oct 2026 14:02:20 +0200 Subject: [PATCH 03/33] add necessary envs and volumes to agent pod --- Cargo.lock | 18 +- Cargo.nix | 18 +- rust/operator-binary/src/controller.rs | 133 +++++++++++- .../src/controller/build/resource/agent.rs | 146 +++++++++++-- .../src/controller/build/security.rs | 120 +---------- .../src/controller/security.rs | 12 ++ .../src/controller/validate.rs | 203 ++++++++++++++++-- rust/shared/src/agent.rs | 13 +- 8 files changed, 482 insertions(+), 181 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a50ea695..f646c56e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1736,7 +1736,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "darling 0.24.1", "regex", @@ -3213,7 +3213,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "const-oid", "ecdsa", @@ -3303,7 +3303,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.119.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "base64 0.23.1", "clap", @@ -3347,7 +3347,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3358,7 +3358,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "jiff", "k8s-openapi", @@ -3375,7 +3375,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "axum", "clap", @@ -3399,7 +3399,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "kube", "schemars", @@ -3413,7 +3413,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "convert_case", "darling 0.24.1", @@ -3430,7 +3430,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 95a0e642..27002f0f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5508,7 +5508,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "k8s_version"; @@ -10506,7 +10506,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_certs"; @@ -10877,7 +10877,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_operator"; @@ -11076,7 +11076,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -11111,7 +11111,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_shared"; @@ -11192,7 +11192,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_telemetry"; @@ -11302,7 +11302,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_versioned"; @@ -11352,7 +11352,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -11416,7 +11416,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_webhook"; diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 98eb8d54..66aedb63 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -15,12 +15,13 @@ use std::{ use const_format::concatcp; use snafu::{ResultExt, Snafu}; +use stackable_kafka_shared::agent::ClientSecurity; use stackable_operator::{ cli::OperatorEnvironmentOptions, cluster_resources::ClusterResourceApplyStrategy, commons::{networking::DomainName, product_image_selection::ResolvedProductImage}, constant, - crd::listener, + crd::{listener, platform_access::v1alpha1::TlsClientCredential}, k8s_openapi::api::{ apps::v1::{Deployment, StatefulSet}, core::v1::{ConfigMap, Service, ServiceAccount}, @@ -40,7 +41,7 @@ use stackable_operator::{ role_group_utils::ResourceNames, role_utils, types::{ - kubernetes::{ConfigMapName, ListenerName, NamespaceName, Uid}, + kubernetes::{ConfigMapName, ListenerName, NamespaceName, SecretClassName, Uid}, operator::{ClusterName, ControllerName, OperatorName, ProductName, ProductVersion}, }, }, @@ -346,7 +347,29 @@ impl ValidatedClusterConfig { /// The configuration of the agent deployed with the cluster. pub struct ValidatedAgentConfig { pub image: String, - pub authentication: v1alpha1::KafkaPlatformAccessAuthentication, + pub connection: AgentConnection, +} + +#[derive(Debug, PartialEq)] +pub enum AgentConnection { + Plaintext, + Tls { + server_ca_secret_class: SecretClassName, + }, + MutualTls { + server_ca_secret_class: SecretClassName, + client_credential: TlsClientCredential, + }, +} + +impl From<&AgentConnection> for ClientSecurity { + fn from(connection: &AgentConnection) -> Self { + match connection { + AgentConnection::Plaintext => Self::Plaintext, + AgentConnection::Tls { .. } => Self::Tls, + AgentConnection::MutualTls { .. } => Self::MutualTls, + } + } } /// Per-role configuration extracted during validation. @@ -529,16 +552,24 @@ pub fn error_policy( #[cfg(test)] pub(crate) mod test_support { - use std::collections::BTreeMap; + use std::{collections::BTreeMap, str::FromStr}; use stackable_operator::{ + builder::meta::ObjectMetaBuilder, cli::OperatorEnvironmentOptions, commons::networking::DomainName, - crd::listener, + crd::{ + authentication::{core, kerberos, tls}, + listener, + }, utils::{cluster_info::KubernetesClusterInfo, yaml_from_str_singleton_map}, + v2::types::kubernetes::SecretClassName, }; - use super::{ValidatedCluster, dereference::DereferencedObjects, validate::validate}; + use super::{ + ValidatedCluster, dereference::DereferencedObjects, security::ValidatedKafkaSecurity, + validate::validate, + }; use crate::crd::{authentication::ResolvedAuthenticationClasses, v1alpha1}; /// The expected `app.kubernetes.io/version` label value for the given product version. @@ -557,6 +588,96 @@ pub(crate) mod test_support { yaml_from_str_singleton_map(yaml).expect("invalid test KafkaCluster YAML") } + fn tls_auth_class() -> core::v1alpha1::AuthenticationClass { + core::v1alpha1::AuthenticationClass { + metadata: ObjectMetaBuilder::new().name("tls-auth").build(), + spec: core::v1alpha1::AuthenticationClassSpec { + provider: core::v1alpha1::AuthenticationClassProvider::Tls( + tls::v1alpha1::AuthenticationProvider { + client_cert_secret_class: Some("client-auth-secret-class".to_string()), + }, + ), + }, + } + } + + fn kerberos_auth_class() -> core::v1alpha1::AuthenticationClass { + core::v1alpha1::AuthenticationClass { + metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), + spec: core::v1alpha1::AuthenticationClassSpec { + provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( + kerberos::v1alpha1::AuthenticationProvider { + kerberos_secret_class: "kerberos-secret-class".to_string(), + }, + ), + }, + } + } + + fn no_auth() -> ResolvedAuthenticationClasses { + ResolvedAuthenticationClasses::new(vec![]) + } + + /// Plaintext: no TLS, no authentication, no OPA. + pub fn plaintext() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + None, + None, + ) + } + + /// Server TLS only (encryption without client authentication). + pub fn server_tls() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + None, + ) + } + + /// Mutual TLS (client-certificate authentication). + pub fn client_auth_tls() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![tls_auth_class()]), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + None, + ) + } + + /// Kerberos, which also requires server and internal TLS. + pub fn kerberos() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![kerberos_auth_class()]), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + None, + ) + } + + /// Internal TLS only (broker/controller encryption without client TLS). + pub fn internal_tls() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + None, + None, + ) + } + + /// OPA authorization with a TLS truststore (internal + server TLS also enabled). + pub fn opa() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + Some("opa-tls".parse().unwrap()), + ) + } + fn cluster_info() -> KubernetesClusterInfo { KubernetesClusterInfo { cluster_domain: DomainName::try_from("cluster.local").expect("valid domain"), diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index bad1fb2a..81c7e34a 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -2,13 +2,22 @@ use std::str::FromStr; +use stackable_kafka_shared::agent::{ + ClientSecurity, DISCOVERY_DIR, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE, + KAFKA_TLS_DIR, +}; use stackable_operator::{ builder::{ meta::ObjectMetaBuilder, pod::{ - PodBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + volume::{SecretFormat, SecretOperatorVolumeSourceBuilder, VolumeBuilder}, }, }, + commons::secret_class::SecretClassVolumeProvisionParts, constant, k8s_openapi::{ api::{ @@ -20,20 +29,22 @@ use stackable_operator::{ }, kube::api::ObjectMeta, v2::{ - builder::{meta::ownerreference_from_resource, pod::container::new_container_builder}, - types::kubernetes::ContainerName, + builder::{ + meta::ownerreference_from_resource, + pod::container::{EnvVarSet, new_container_builder}, + }, + types::kubernetes::{ContainerName, SecretClassName, VolumeName}, }, }; -use crate::{ - controller::{ - ValidatedAgentConfig, ValidatedCluster, - build::{agent_selector, recommended_labels_for_agent_resources}, - }, - crd::v1alpha1::KafkaPlatformAccessAuthentication, +use crate::controller::{ + AgentConnection, ValidatedAgentConfig, ValidatedCluster, + build::{agent_selector, recommended_labels_for_agent_resources}, }; constant!(AGENT_CONTAINER_NAME: ContainerName = "agent"); +constant!(DISCOVERY_VOLUME_NAME: VolumeName = "discovery"); +constant!(KAFKA_TLS_VOLUME_NAME: VolumeName = "tls-kafka"); pub const AGENT_CLUSTER_ROLE_NAME: &str = "kafka-agent-clusterrole"; pub fn build_agent_deployment( @@ -43,16 +54,35 @@ pub fn build_agent_deployment( let mut cb_agent = new_container_builder(&AGENT_CONTAINER_NAME); let mut pod_builder = PodBuilder::new(); - match &agent_config.authentication { - KafkaPlatformAccessAuthentication::Tls(credential) => { - credential.add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]) + add_discovery_volume_and_mount(cluster, &mut pod_builder, &mut cb_agent); + match &agent_config.connection { + AgentConnection::Plaintext => {} + AgentConnection::Tls { + server_ca_secret_class, + } => { + add_kafka_tls_volume_and_mount(server_ca_secret_class, &mut pod_builder, &mut cb_agent) + } + AgentConnection::MutualTls { + server_ca_secret_class, + client_credential, + } => { + add_kafka_tls_volume_and_mount(server_ca_secret_class, &mut pod_builder, &mut cb_agent); + client_credential.add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]); } - KafkaPlatformAccessAuthentication::Anonymous(_) => {} } cb_agent .image(&agent_config.image) .args(vec!["run".to_owned()]) + .add_env_vars( + EnvVarSet::new() + .with_value(&ENV_CLUSTER_NAME, cluster.name.to_string()) + .with_value(&ENV_CLUSTER_NAMESPACE, cluster.namespace.to_string()) + .with_value( + &ENV_CLIENT_SECURITY, + ClientSecurity::from(&agent_config.connection).to_string(), + ), + ) .add_env_var_from_source( "KUBERNETES_NODE_NAME", EnvVarSource { @@ -128,6 +158,48 @@ pub fn build_agent_role_binding(cluster: &ValidatedCluster) -> RoleBinding { } } +fn add_discovery_volume_and_mount( + cluster: &ValidatedCluster, + pod_builder: &mut PodBuilder, + container_builder: &mut ContainerBuilder, +) { + pod_builder + .add_volume( + VolumeBuilder::new(&*DISCOVERY_VOLUME_NAME) + .with_config_map(cluster.name.to_string()) + .build(), + ) + .expect("the volume name is statically defined and must not collide"); + container_builder + .add_volume_mount(&*DISCOVERY_VOLUME_NAME, DISCOVERY_DIR) + .expect("the mount path is statically defined and must not collide"); +} + +fn add_kafka_tls_volume_and_mount( + secret_class: &SecretClassName, + pod_builder: &mut PodBuilder, + container_builder: &mut ContainerBuilder, +) { + pod_builder + .add_volume( + VolumeBuilder::new(&*KAFKA_TLS_VOLUME_NAME) + .ephemeral( + SecretOperatorVolumeSourceBuilder::new( + secret_class, + SecretClassVolumeProvisionParts::Public, + ) + .with_format(SecretFormat::TlsPem) + .build() + .expect("the annotations are built from a valid SecretClass name"), + ) + .build(), + ) + .expect("the volume name is statically defined and must not collide"); + container_builder + .add_volume_mount(&*KAFKA_TLS_VOLUME_NAME, KAFKA_TLS_DIR) + .expect("the mount path is statically defined and must not collide"); +} + /// The name of all agent resources. Kinds don't share a namespace, so unlike a suffix per kind, this /// cannot collide with the resources of a cluster named `-agent`. fn agent_name(cluster: &ValidatedCluster) -> String { @@ -240,12 +312,37 @@ mod tests { assert_eq!(pod_spec["containers"][0]["args"], json!(["run"])); assert_eq!( pod_spec["containers"][0]["volumeMounts"], - json!([{"mountPath": "/stackable/secrets/tls-client-cert", "name": "tls-client-cert"}]) + json!([ + {"mountPath": "/stackable/discovery", "name": "discovery"}, + {"mountPath": "/stackable/secrets/tls-kafka", "name": "tls-kafka"}, + {"mountPath": "/stackable/secrets/tls-platform-access", "name": "tls-platform-access"} + ]) ); - assert_eq!(pod_spec["volumes"][0]["name"], "tls-client-cert"); assert_eq!( - pod_spec["volumes"][0]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"]["secrets.stackable.tech/class"], - "tls" + pod_spec["volumes"][0]["configMap"], + json!({"name": "simple-kafka"}) + ); + assert_eq!( + pod_spec["volumes"][1]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"], + json!({ + "secrets.stackable.tech/class": "tls", + "secrets.stackable.tech/format": "tls-pem", + "secrets.stackable.tech/provision-parts": "public" + }) + ); + + assert_eq!( + pod_spec["containers"][0]["env"], + json!([ + {"name": "KAFKA_CLIENT_SECURITY", "value": "mutual-tls"}, + {"name": "KAFKA_CLUSTER_NAME", "value": "simple-kafka"}, + {"name": "KAFKA_CLUSTER_NAMESPACE", "value": "default"}, + { + "name": "KUBERNETES_NODE_NAME", + "valueFrom": {"fieldRef": {"fieldPath": "spec.nodeName"}} + }, + {"name": "KUBERNETES_CLUSTER_DOMAIN", "value": "cluster.local"} + ]) ); } @@ -255,18 +352,23 @@ mod tests { agent_deployment("{enabled: true, authentication: {tls: {secret: my-cert}}}"); assert_eq!( - deployment["spec"]["template"]["spec"]["volumes"], - json!([{"name": "tls-client-cert", "secret": {"secretName": "my-cert"}}]) + deployment["spec"]["template"]["spec"]["volumes"][2], + json!({"name": "tls-platform-access", "secret": {"secretName": "my-cert"}}) ); } #[test] - fn deployment_with_anonymous_authentication_mounts_no_credential() { + fn deployment_with_anonymous_authentication_verifies_brokers_without_credential() { let deployment = agent_deployment("{enabled: true, authentication: {anonymous: {}}}"); let pod_spec = &deployment["spec"]["template"]["spec"]; - assert_eq!(pod_spec["volumes"], Value::Null); - assert_eq!(pod_spec["containers"][0]["volumeMounts"], Value::Null); + assert_eq!(pod_spec["volumes"][0]["name"], "discovery"); + assert_eq!(pod_spec["volumes"][1]["name"], "tls-kafka"); + assert_eq!(pod_spec["volumes"][2], Value::Null); + assert_eq!( + pod_spec["containers"][0]["env"][0], + json!({"name": "KAFKA_CLIENT_SECURITY", "value": "tls"}) + ); } #[test] diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index b9eb650a..cdab2d9e 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -17,7 +17,6 @@ use stackable_operator::{ }, commons::secret_class::SecretClassVolumeProvisionParts, constant, - crd::authentication::core, k8s_openapi::api::core::v1::Volume, shared::time::Duration, v2::types::kubernetes::VolumeName, @@ -250,7 +249,7 @@ pub fn add_broker_volume_and_volume_mounts( requested_secret_lifetime: &Duration, ) -> Result<(), Error> { // add tls (server or client authentication volumes) if required - if let Some(tls_server_secret_class) = tls_secret_class(security) { + if let Some(tls_server_secret_class) = security.tls_secret_class() { // We have to mount tls pem files for kcat's readiness-probe command (the mount can be // used directly) pod_builder @@ -572,19 +571,6 @@ pub fn controller_config_settings(security: &ValidatedKafkaSecurity) -> BTreeMap config } -/// Returns the `SecretClass` provided in a `AuthenticationClass` for TLS. -fn tls_secret_class(security: &ValidatedKafkaSecurity) -> Option<&str> { - security - .tls_client_authentication_class() - .and_then(|auth_class| match &auth_class.spec.provider { - core::v1alpha1::AuthenticationClassProvider::Tls(tls) => { - tls.client_cert_secret_class.as_deref() - } - _ => None, - }) - .or_else(|| security.tls_server_secret_class()) -} - /// Creates ephemeral volumes to mount the `SecretClass` into the Pods for kcat client fn create_kcat_tls_volume( volume_name: &VolumeName, @@ -676,107 +662,13 @@ fn kcat_client_sasl_ssl(cert_directory: &str, service_name: &str) -> Vec } #[cfg(test)] -pub(crate) mod tests { - use std::{collections::BTreeMap, str::FromStr}; - - use stackable_operator::{ - builder::meta::ObjectMetaBuilder, - crd::authentication::{core, kerberos, tls}, - v2::types::kubernetes::SecretClassName, - }; +mod tests { + use std::collections::BTreeMap; use super::*; - use crate::crd::authentication::ResolvedAuthenticationClasses; - - fn tls_auth_class() -> core::v1alpha1::AuthenticationClass { - core::v1alpha1::AuthenticationClass { - metadata: ObjectMetaBuilder::new().name("tls-auth").build(), - spec: core::v1alpha1::AuthenticationClassSpec { - provider: core::v1alpha1::AuthenticationClassProvider::Tls( - tls::v1alpha1::AuthenticationProvider { - client_cert_secret_class: Some("client-auth-secret-class".to_string()), - }, - ), - }, - } - } - - fn kerberos_auth_class() -> core::v1alpha1::AuthenticationClass { - core::v1alpha1::AuthenticationClass { - metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), - spec: core::v1alpha1::AuthenticationClassSpec { - provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( - kerberos::v1alpha1::AuthenticationProvider { - kerberos_secret_class: "kerberos-secret-class".to_string(), - }, - ), - }, - } - } - - fn no_auth() -> ResolvedAuthenticationClasses { - ResolvedAuthenticationClasses::new(vec![]) - } - - /// Plaintext: no TLS, no authentication, no OPA. - fn plaintext() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - None, - None, - ) - } - - /// Server TLS only (encryption without client authentication). - fn server_tls() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - None, - ) - } - - /// Mutual TLS (client-certificate authentication). - fn client_auth_tls() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![tls_auth_class()]), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - None, - ) - } - - /// Kerberos, which also requires server and internal TLS. - pub(crate) fn kerberos() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![kerberos_auth_class()]), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - None, - ) - } - - /// Internal TLS only (broker/controller encryption without client TLS). - fn internal_tls() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - None, - None, - ) - } - - /// OPA authorization with a TLS truststore (internal + server TLS also enabled). - fn opa() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - Some("opa-tls".parse().unwrap()), - ) - } + use crate::controller::test_support::{ + client_auth_tls, internal_tls, kerberos, opa, plaintext, server_tls, + }; fn as_str_vec(commands: &[String]) -> Vec<&str> { commands.iter().map(String::as_str).collect() diff --git a/rust/operator-binary/src/controller/security.rs b/rust/operator-binary/src/controller/security.rs index 90bd6217..a97c732f 100644 --- a/rust/operator-binary/src/controller/security.rs +++ b/rust/operator-binary/src/controller/security.rs @@ -101,6 +101,18 @@ impl ValidatedKafkaSecurity { .get_tls_authentication_class() } + /// The `SecretClass` of a TLS `AuthenticationClass`, otherwise the server `SecretClass`. + pub fn tls_secret_class(&self) -> Option<&str> { + self.tls_client_authentication_class() + .and_then(|auth_class| match &auth_class.spec.provider { + core::v1alpha1::AuthenticationClassProvider::Tls(tls) => { + tls.client_cert_secret_class.as_deref() + } + _ => None, + }) + .or_else(|| self.tls_server_secret_class()) + } + /// Retrieve the internal `SecretClass`. Internal (inter-broker) TLS is mandatory, so this is /// always set (defaults to `tls`). pub fn tls_internal_secret_class(&self) -> &str { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 62fb2bff..f500b059 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -6,7 +6,7 @@ use std::{collections::BTreeMap, str::FromStr}; use serde::Serialize; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu, ensure}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection, @@ -24,14 +24,15 @@ use stackable_operator::{ validate_logging_configuration_for_container, }, role_utils::{JavaCommonConfig, Role, with_validated_config}, - types::kubernetes::ConfigMapName, + types::kubernetes::{ConfigMapName, SecretClassName}, }, }; use crate::{ controller::{ - RoleGroupName, ValidatedAgentConfig, ValidatedCluster, ValidatedClusterConfig, - ValidatedKafkaConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, + AgentConnection, RoleGroupName, ValidatedAgentConfig, ValidatedCluster, + ValidatedClusterConfig, ValidatedKafkaConfig, ValidatedRoleConfig, + ValidatedRoleGroupConfig, dereference::DereferencedObjects, security::{self, ValidatedKafkaSecurity}, }, @@ -115,6 +116,28 @@ pub enum Error { to join without it" ))] MissingKraftControllerRole, + + #[snafu(display( + "platform access does not support the cluster authentication method {authentication:?}" + ))] + UnsupportedClusterAuthentication { authentication: String }, + + #[snafu(display( + "the cluster requires client authentication, but platform access is anonymous" + ))] + AnonymousPlatformAccessWithClientAuthentication, + + #[snafu(display( + "the platform access authentication method {authentication:?} requires the cluster to use \ + TLS" + ))] + PlatformAccessRequiresTls { authentication: String }, + + #[snafu(display("the SecretClass {secret_class:?} of the client listener is invalid"))] + ParseClientListenerSecretClass { + source: stackable_operator::v2::macros::attributed_string_type::Error, + secret_class: String, + }, } /// Validated logging configuration for a Kafka role group's Kafka and (optional) Vector @@ -334,15 +357,16 @@ pub fn validate( .cluster_domain .clone(); - let agent_config = kafka - .spec - .platform_access - .as_ref() - .filter(|platform_access| platform_access.enabled) - .map(|platform_access| ValidatedAgentConfig { + let agent_config = match &kafka.spec.platform_access { + Some(platform_access) if platform_access.enabled => Some(ValidatedAgentConfig { image: agent_image.to_owned(), - authentication: platform_access.authentication.clone(), - }); + connection: validate_agent_connection( + &platform_access.authentication, + &kafka_security, + )?, + }), + _ => None, + }; Ok(ValidatedCluster::new( name, @@ -452,15 +476,164 @@ fn inject_cluster_id(env_overrides: EnvVarSet, cluster_id: Option<&str>) -> EnvV } } +fn validate_agent_connection( + authentication: &v1alpha1::KafkaPlatformAccessAuthentication, + security: &ValidatedKafkaSecurity, +) -> Result { + ensure!( + !security.has_kerberos_enabled(), + UnsupportedClusterAuthenticationSnafu { + authentication: "kerberos" + } + ); + + let server_ca_secret_class = security + .tls_secret_class() + .map(|secret_class| { + SecretClassName::from_str(secret_class) + .context(ParseClientListenerSecretClassSnafu { secret_class }) + }) + .transpose()?; + + match (authentication, server_ca_secret_class) { + (v1alpha1::KafkaPlatformAccessAuthentication::Anonymous(_), None) => { + Ok(AgentConnection::Plaintext) + } + ( + v1alpha1::KafkaPlatformAccessAuthentication::Anonymous(_), + Some(server_ca_secret_class), + ) => { + ensure!( + security.tls_client_authentication_class().is_none(), + AnonymousPlatformAccessWithClientAuthenticationSnafu + ); + Ok(AgentConnection::Tls { + server_ca_secret_class, + }) + } + (v1alpha1::KafkaPlatformAccessAuthentication::Tls(_), None) => { + PlatformAccessRequiresTlsSnafu { + authentication: "tls", + } + .fail() + } + ( + v1alpha1::KafkaPlatformAccessAuthentication::Tls(client_credential), + Some(server_ca_secret_class), + ) => Ok(AgentConnection::MutualTls { + server_ca_secret_class, + client_credential: client_credential.clone(), + }), + } +} + #[cfg(test)] mod tests { use std::str::FromStr; - use stackable_operator::v2::{ - builder::pod::container::EnvVarSet, types::operator::RoleGroupName, + use rstest::rstest; + use stackable_operator::{ + crd::platform_access::v1alpha1::{Anonymous, TlsClientCredential}, + v2::{ + builder::pod::container::EnvVarSet, + types::{ + kubernetes::{SecretClassName, SecretName}, + operator::RoleGroupName, + }, + }, }; - use super::{Error, KAFKA_CLUSTER_ID, inject_cluster_id}; + use super::{Error, KAFKA_CLUSTER_ID, inject_cluster_id, validate_agent_connection}; + use crate::{ + controller::{ + AgentConnection, + security::ValidatedKafkaSecurity, + test_support::{client_auth_tls, kerberos, plaintext, server_tls}, + }, + crd::v1alpha1::KafkaPlatformAccessAuthentication, + }; + + fn anonymous() -> KafkaPlatformAccessAuthentication { + KafkaPlatformAccessAuthentication::Anonymous(Anonymous::default()) + } + + fn agent_cert() -> TlsClientCredential { + TlsClientCredential::Secret( + SecretName::from_str("agent-cert").expect("the Secret name should be valid"), + ) + } + + fn tls() -> KafkaPlatformAccessAuthentication { + KafkaPlatformAccessAuthentication::Tls(agent_cert()) + } + + fn tls_secret_class(name: &str) -> SecretClassName { + SecretClassName::from_str(name).expect("the SecretClass name should be valid") + } + + #[rstest] + #[case::plaintext_anonymous(plaintext(), anonymous(), AgentConnection::Plaintext)] + #[case::server_tls_anonymous( + server_tls(), + anonymous(), + AgentConnection::Tls { server_ca_secret_class: tls_secret_class("tls") } + )] + #[case::server_tls_with_credential( + server_tls(), + tls(), + AgentConnection::MutualTls { + server_ca_secret_class: tls_secret_class("tls"), + client_credential: agent_cert(), + } + )] + #[case::client_auth_with_credential( + client_auth_tls(), + tls(), + AgentConnection::MutualTls { + server_ca_secret_class: tls_secret_class("client-auth-secret-class"), + client_credential: agent_cert(), + } + )] + fn agent_connection( + #[case] security: ValidatedKafkaSecurity, + #[case] authentication: KafkaPlatformAccessAuthentication, + #[case] expected: AgentConnection, + ) { + let connection = validate_agent_connection(&authentication, &security) + .expect("the agent connection should be valid"); + + assert_eq!(expected, connection); + } + + #[test] + fn reject_unsupported_cluster_authentication() { + let error = validate_agent_connection(&tls(), &kerberos()) + .expect_err("the agent connection should be invalid"); + + assert!(matches!( + error, + Error::UnsupportedClusterAuthentication { .. } + )); + } + + #[test] + fn reject_anonymous_agent_connection_with_client_authentication() { + let error = validate_agent_connection(&anonymous(), &client_auth_tls()) + .expect_err("the agent connection should be invalid"); + + assert!(matches!( + error, + Error::AnonymousPlatformAccessWithClientAuthentication + )); + } + + #[test] + fn reject_agent_connection_requiring_tls_without_tls() { + let error = validate_agent_connection(&tls(), &plaintext()) + .expect_err("the agent connection should be invalid"); + + assert!(matches!(error, Error::PlatformAccessRequiresTls { .. })); + } use crate::{ controller::test_support::{app_version_label, minimal_kafka, validated_cluster}, crd::role::KafkaRole, diff --git a/rust/shared/src/agent.rs b/rust/shared/src/agent.rs index 4af72434..61736017 100644 --- a/rust/shared/src/agent.rs +++ b/rust/shared/src/agent.rs @@ -1,17 +1,18 @@ //! Paths and environment variables through which the operator configures the agent. +use std::str::FromStr; + +use stackable_operator::{constant, v2::builder::pod::container::EnvVarName}; use strum::{Display, EnumString}; pub const DISCOVERY_DIR: &str = "/stackable/discovery"; /// Directory of the CA certificate the Kafka brokers are verified with. -pub const SERVER_CA_DIR: &str = "/stackable/secrets/tls-server-ca"; - -pub const ENV_CLUSTER_NAME: &str = "KAFKA_CLUSTER_NAME"; - -pub const ENV_CLUSTER_NAMESPACE: &str = "KAFKA_CLUSTER_NAMESPACE"; +pub const KAFKA_TLS_DIR: &str = "/stackable/secrets/tls-kafka"; -pub const ENV_CLIENT_SECURITY: &str = "KAFKA_CLIENT_SECURITY"; +constant!(pub ENV_CLUSTER_NAME: EnvVarName = "KAFKA_CLUSTER_NAME"); +constant!(pub ENV_CLUSTER_NAMESPACE: EnvVarName = "KAFKA_CLUSTER_NAMESPACE"); +constant!(pub ENV_CLIENT_SECURITY: EnvVarName = "KAFKA_CLIENT_SECURITY"); /// How the agent connects to the brokers. #[derive(Clone, Copy, Debug, Display, EnumString, Eq, PartialEq)] From 9ba595dcc6eacc0b7e7f276d6d4e8e7661df4d84 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 Oct 2026 11:30:09 +0200 Subject: [PATCH 04/33] reject zookeeper-mode when platformAccess is configured --- .../src/controller/build/resource/agent.rs | 6 +- .../src/controller/validate.rs | 60 ++++++++++++++++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index 81c7e34a..20c6323a 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -241,8 +241,12 @@ mod tests { spec: image: {image} clusterConfig: - zookeeperConfigMapName: xyz + metadataManager: kraft platformAccess: {platform_access} + controllers: + roleGroups: + default: + replicas: 1 brokers: roleGroups: default: diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index f500b059..ce9497f0 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -122,6 +122,9 @@ pub enum Error { ))] UnsupportedClusterAuthentication { authentication: String }, + #[snafu(display("platform access is only supported in KRaft mode"))] + PlatformAccessRequiresKraft, + #[snafu(display( "the cluster requires client authentication, but platform access is anonymous" ))] @@ -358,13 +361,19 @@ pub fn validate( .clone(); let agent_config = match &kafka.spec.platform_access { - Some(platform_access) if platform_access.enabled => Some(ValidatedAgentConfig { - image: agent_image.to_owned(), - connection: validate_agent_connection( - &platform_access.authentication, - &kafka_security, - )?, - }), + Some(platform_access) if platform_access.enabled => { + ensure!( + metadata_manager == crate::crd::MetadataManager::KRaft, + PlatformAccessRequiresKraftSnafu + ); + Some(ValidatedAgentConfig { + image: agent_image.to_owned(), + connection: validate_agent_connection( + &platform_access.authentication, + &kafka_security, + )?, + }) + } _ => None, }; @@ -929,4 +938,41 @@ mod tests { let _cluster = validated_cluster(&kafka); } + + #[test] + fn zookeeper_mode_rejects_platform_access() { + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + zookeeperConfigMapName: zk-discovery + platformAccess: + enabled: true + authentication: + anonymous: {} + brokers: + roleGroups: + default: + replicas: 3 + "#, + ); + + let result = crate::controller::test_support::validate_err(&kafka); + let Err(error) = result else { + panic!("validate should reject platform access in ZooKeeper mode"); + }; + + assert!( + matches!(error, Error::PlatformAccessRequiresKraft), + "expected PlatformAccessRequiresKraft, got: {error:?}" + ); + } } From 2c730cad26e55395472a6c008f038e38deaa6ef1 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 15 Oct 2026 19:21:47 +0200 Subject: [PATCH 05/33] add config to agent: bootstrap servers from discovery cm, security mode, tls paths --- Cargo.lock | 5 ++ Cargo.nix | 22 +++++ rust/agent-binary/Cargo.toml | 7 ++ rust/agent-binary/src/config.rs | 141 ++++++++++++++++++++++++++++++++ rust/agent-binary/src/main.rs | 46 ++++++++++- 5 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 rust/agent-binary/src/config.rs diff --git a/Cargo.lock b/Cargo.lock index f646c56e..bfb174c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3258,6 +3258,11 @@ dependencies = [ "anyhow", "built", "clap", + "const_format", + "rstest", + "snafu 0.9.2", + "stackable-kafka-admin", + "stackable-kafka-shared", "stackable-operator", "tokio", "tracing", diff --git a/Cargo.nix b/Cargo.nix index 27002f0f..15ce6ebb 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10698,6 +10698,22 @@ rec { name = "clap"; packageId = "clap"; } + { + name = "const_format"; + packageId = "const_format"; + } + { + name = "snafu"; + packageId = "snafu 0.9.2"; + } + { + name = "stackable-kafka-admin"; + packageId = "stackable-kafka-admin"; + } + { + name = "stackable-kafka-shared"; + packageId = "stackable-kafka-shared"; + } { name = "stackable-operator"; packageId = "stackable-operator"; @@ -10720,6 +10736,12 @@ rec { features = [ "chrono" "git2" ]; } ]; + devDependencies = [ + { + name = "rstest"; + packageId = "rstest"; + } + ]; }; "stackable-kafka-operator" = rec { diff --git a/rust/agent-binary/Cargo.toml b/rust/agent-binary/Cargo.toml index c138e675..925327a2 100644 --- a/rust/agent-binary/Cargo.toml +++ b/rust/agent-binary/Cargo.toml @@ -9,13 +9,20 @@ repository.workspace = true publish = false [dependencies] +stackable-kafka-admin.workspace = true +stackable-kafka-shared.workspace = true stackable-operator.workspace = true anyhow.workspace = true clap.workspace = true +const_format.workspace = true +snafu.workspace = true tokio.workspace = true tracing.workspace = true +[dev-dependencies] +rstest.workspace = true + [build-dependencies] built.workspace = true diff --git a/rust/agent-binary/src/config.rs b/rust/agent-binary/src/config.rs new file mode 100644 index 00000000..bdb2bc9f --- /dev/null +++ b/rust/agent-binary/src/config.rs @@ -0,0 +1,141 @@ +//! The agent's view of the files and settings the operator provides. + +use std::{ + fs, io, + path::{Path, PathBuf}, +}; + +use const_format::concatcp; +use snafu::{ResultExt, Snafu}; +use stackable_kafka_admin::{ + bootstrap::{self, BrokerAddress, parse_bootstrap_servers}, + connection::Security, +}; +use stackable_kafka_shared::agent::{ClientSecurity, DISCOVERY_DIR, KAFKA_TLS_DIR}; +use stackable_operator::crd::platform_access::tls::MOUNT_PATH; + +/// The key `KAFKA` of the discovery ConfigMap, holding `host:port,host:port` +pub const BOOTSTRAP_SERVERS_FILE: &str = concatcp!(DISCOVERY_DIR, "/KAFKA"); + +const CA_FILE: &str = concatcp!(KAFKA_TLS_DIR, "/ca.crt"); +const CERTIFICATE_FILE: &str = concatcp!(MOUNT_PATH, "/tls.crt"); +const KEY_FILE: &str = concatcp!(MOUNT_PATH, "/tls.key"); + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to read the bootstrap servers from {path:?}"))] + ReadBootstrapServers { source: io::Error, path: PathBuf }, + + #[snafu(display("the bootstrap servers in {path:?} are invalid"))] + ParseBootstrapServers { + source: bootstrap::Error, + path: PathBuf, + }, +} + +pub fn security(client_security: ClientSecurity) -> Security { + match client_security { + ClientSecurity::Plaintext => Security::Plaintext, + ClientSecurity::Tls => Security::Tls { ca: CA_FILE.into() }, + ClientSecurity::MutualTls => Security::MutualTls { + ca: CA_FILE.into(), + certificate: CERTIFICATE_FILE.into(), + key: KEY_FILE.into(), + }, + } +} + +/// Returns `None` while the operator has written no bootstrap servers to the discovery configmap yet. +pub fn read_bootstrap_servers(path: &Path) -> Result>, Error> { + let value = fs::read_to_string(path).context(ReadBootstrapServersSnafu { path })?; + if value.trim().is_empty() { + return Ok(None); + } + parse_bootstrap_servers(&value) + .map(Some) + .context(ParseBootstrapServersSnafu { path }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::plaintext(ClientSecurity::Plaintext, Security::Plaintext)] + #[case::tls( + ClientSecurity::Tls, + Security::Tls { + ca: "/stackable/secrets/tls-kafka/ca.crt".into(), + } + )] + #[case::mutual_tls( + ClientSecurity::MutualTls, + Security::MutualTls { + ca: "/stackable/secrets/tls-kafka/ca.crt".into(), + certificate: "/stackable/secrets/tls-platform-access/tls.crt".into(), + key: "/stackable/secrets/tls-platform-access/tls.key".into(), + } + )] + fn map_client_security(#[case] client_security: ClientSecurity, #[case] expected: Security) { + assert_eq!(expected, security(client_security)); + } + + #[rstest] + #[case::empty("", None)] + #[case::blank(" \n", None)] + #[case::servers( + "kafka-0:9093,kafka-1:9093", + Some(vec![ + BrokerAddress { host: "kafka-0".to_owned(), port: 9093 }, + BrokerAddress { host: "kafka-1".to_owned(), port: 9093 }, + ]) + )] + fn read_written_bootstrap_servers( + #[case] content: &str, + #[case] expected: Option>, + ) { + let path = temp_file(content); + + let bootstrap_servers = read_bootstrap_servers(&path); + fs::remove_file(&path).expect("the file should be removable"); + + assert_eq!( + expected, + bootstrap_servers.expect("the bootstrap servers should be readable") + ); + } + + #[test] + fn reject_missing_bootstrap_servers_file() { + let path = std::env::temp_dir().join("missing-bootstrap-servers"); + + assert!(matches!( + read_bootstrap_servers(&path).expect_err("a missing file should be an error"), + Error::ReadBootstrapServers { .. } + )); + } + + #[test] + fn reject_invalid_bootstrap_servers() { + let path = temp_file("kafka"); + + let error = read_bootstrap_servers(&path); + fs::remove_file(&path).expect("the file should be removable"); + + assert!(matches!( + error.expect_err("the bootstrap servers should be invalid"), + Error::ParseBootstrapServers { .. } + )); + } + + /// Writes the content to a file unique to this test thread. + fn temp_file(content: &str) -> PathBuf { + let thread = std::thread::current(); + let name = thread.name().unwrap_or("test").replace("::", "-"); + let path = std::env::temp_dir().join(format!("bootstrap-{}-{name}", std::process::id())); + fs::write(&path, content).expect("the file should be writable"); + path + } +} diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 0934f803..df9393ba 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -1,12 +1,20 @@ //! The Stackable Kafka agent: a per-cluster controller that reconciles in-cluster resources like `KafkaTopic`s. +use std::path::Path; + use clap::Parser; +use stackable_kafka_shared::agent::{ + ClientSecurity, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE, +}; use stackable_operator::{ cli::{Command, CommonOptions}, telemetry::Tracing, utils::signal::SignalWatcher, + v2::types::{kubernetes::NamespaceName, operator::ClusterName}, }; +mod config; + mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } @@ -15,14 +23,37 @@ mod built_info { #[clap(about, author)] struct Opts { #[clap(subcommand)] - cmd: Command, + cmd: Command, +} + +#[derive(clap::Parser)] +struct KafkaAgentRun { + /// Name of the KafkaCluster this agent belongs to. + #[arg(long, env = AsRef::::as_ref(&*ENV_CLUSTER_NAME))] + cluster_name: ClusterName, + + /// Namespace of the KafkaCluster. + #[arg(long, env = AsRef::::as_ref(&*ENV_CLUSTER_NAMESPACE))] + cluster_namespace: NamespaceName, + + /// How to connect to the brokers: plaintext, tls or mutual-tls. + #[arg(long, env = AsRef::::as_ref(&*ENV_CLIENT_SECURITY))] + client_security: ClientSecurity, + + #[clap(flatten)] + common: CommonOptions, } #[tokio::main] async fn main() -> anyhow::Result<()> { match Opts::parse().cmd { Command::Crd => anyhow::bail!("this agent has no CRDs, they are owned by the operator"), - Command::Run(CommonOptions { telemetry, .. }) => { + Command::Run(KafkaAgentRun { + cluster_name, + cluster_namespace, + client_security, + common: CommonOptions { telemetry, .. }, + }) => { let _tracing_guard = Tracing::pre_configured(built_info::PKG_NAME, telemetry).init()?; tracing::info!( @@ -35,6 +66,17 @@ async fn main() -> anyhow::Result<()> { description = built_info::PKG_DESCRIPTION ); + let security = config::security(client_security); + let bootstrap_servers = + config::read_bootstrap_servers(Path::new(config::BOOTSTRAP_SERVERS_FILE)); + tracing::info!( + %cluster_name, + %cluster_namespace, + ?security, + ?bootstrap_servers, + "Configured the agent" + ); + let sigterm_watcher = SignalWatcher::sigterm()?; sigterm_watcher.handle().await; } From 5463640b82544b6c3831519da765d35e01d93716 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 Oct 2026 11:33:15 +0200 Subject: [PATCH 06/33] add stackable-kafka-adminto workspace dependencies --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index dfe32d4b..dabfa6f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["crds", "webhook"] } +stackable-kafka-admin = { path = "rust/kafka-admin" } stackable-kafka-shared = { path = "rust/shared" } anyhow = "1.0" From e61fcdbf5be15d7b0b6945710360c5f8771796bd Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 21 Oct 2026 13:56:40 +0200 Subject: [PATCH 07/33] add kafka controller --- Cargo.lock | 3 + Cargo.nix | 13 + .../templates/clusterrole-agent.yaml | 24 + extra/crds.yaml | 25 + rust/agent-binary/Cargo.toml | 3 + rust/agent-binary/src/config.rs | 2 +- rust/agent-binary/src/main.rs | 82 +- rust/agent-binary/src/topic/condition.rs | 302 ++++++++ rust/agent-binary/src/topic/controller.rs | 702 ++++++++++++++++++ .../src/topic/managed_config_keys.rs | 69 ++ rust/agent-binary/src/topic/mod.rs | 8 + rust/agent-binary/src/topic/plan.rs | 324 ++++++++ rust/agent-binary/src/topic/test_support.rs | 23 + rust/shared/src/crd/topic.rs | 18 + 14 files changed, 1589 insertions(+), 9 deletions(-) create mode 100644 rust/agent-binary/src/topic/condition.rs create mode 100644 rust/agent-binary/src/topic/controller.rs create mode 100644 rust/agent-binary/src/topic/managed_config_keys.rs create mode 100644 rust/agent-binary/src/topic/mod.rs create mode 100644 rust/agent-binary/src/topic/plan.rs create mode 100644 rust/agent-binary/src/topic/test_support.rs diff --git a/Cargo.lock b/Cargo.lock index bfb174c5..10295ba6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3259,11 +3259,14 @@ dependencies = [ "built", "clap", "const_format", + "futures", "rstest", + "serde_json", "snafu 0.9.2", "stackable-kafka-admin", "stackable-kafka-shared", "stackable-operator", + "strum", "tokio", "tracing", ] diff --git a/Cargo.nix b/Cargo.nix index 15ce6ebb..32a7143c 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10702,6 +10702,14 @@ rec { name = "const_format"; packageId = "const_format"; } + { + name = "futures"; + packageId = "futures"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } { name = "snafu"; packageId = "snafu 0.9.2"; @@ -10719,6 +10727,11 @@ rec { packageId = "stackable-operator"; features = [ "crds" "webhook" ]; } + { + name = "strum"; + packageId = "strum"; + features = [ "derive" ]; + } { name = "tokio"; packageId = "tokio"; diff --git a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml index a1701506..e138ef01 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml @@ -8,6 +8,30 @@ metadata: labels: {{- include "kafka-operator.labels" . | nindent 4 }} rules: + # KafkaTopics of the agent's cluster: watched, annotated (ownership, managed config keys) and finalizers. + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics + verbs: + - get + - list + - watch + - patch + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics/status + verbs: + - patch + # Required to report reconciliation errors as events on the KafkaTopic objects. + - apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - patch {{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} # On OpenShift, the agent pod must be allowed to use the nonroot-v2 SCC - apiGroups: diff --git a/extra/crds.yaml b/extra/crds.yaml index 3c824878..a778f97b 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2365,6 +2365,31 @@ spec: - type type: object type: array + partitions: + default: [] + description: The partitions of the topic in Kafka. + items: + properties: + index: + format: int32 + type: integer + replicas: + description: |- + Broker ids holding a replica, the preferred leader first. During a reassignment, this + includes the replicas being added and removed. + items: + format: int32 + type: integer + type: array + required: + - index + - replicas + type: object + type: array + topicId: + description: The topic id in Kafka, as shown by `kafka-topics.sh`. + nullable: true + type: string type: object required: - spec diff --git a/rust/agent-binary/Cargo.toml b/rust/agent-binary/Cargo.toml index 925327a2..eb1fe94c 100644 --- a/rust/agent-binary/Cargo.toml +++ b/rust/agent-binary/Cargo.toml @@ -16,7 +16,10 @@ stackable-operator.workspace = true anyhow.workspace = true clap.workspace = true const_format.workspace = true +futures.workspace = true +serde_json.workspace = true snafu.workspace = true +strum.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/rust/agent-binary/src/config.rs b/rust/agent-binary/src/config.rs index bdb2bc9f..41cdf48a 100644 --- a/rust/agent-binary/src/config.rs +++ b/rust/agent-binary/src/config.rs @@ -45,7 +45,7 @@ pub fn security(client_security: ClientSecurity) -> Security { } } -/// Returns `None` while the operator has written no bootstrap servers to the discovery configmap yet. +/// Returns `None` while the operator has written no bootstrap servers to the discovery ConfigMap yet. pub fn read_bootstrap_servers(path: &Path) -> Result>, Error> { let value = fs::read_to_string(path).context(ReadBootstrapServersSnafu { path })?; if value.trim().is_empty() { diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index df9393ba..b0bbe1ae 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -1,19 +1,38 @@ //! The Stackable Kafka agent: a per-cluster controller that reconciles in-cluster resources like `KafkaTopic`s. -use std::path::Path; +#![allow(clippy::large_enum_variant, clippy::result_large_err)] + +use std::sync::Arc; use clap::Parser; -use stackable_kafka_shared::agent::{ - ClientSecurity, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE, +use futures::StreamExt; +use stackable_kafka_shared::{ + agent::{ClientSecurity, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE}, + crd::topic::v1alpha1::KafkaTopic, }; use stackable_operator::{ cli::{Command, CommonOptions}, + client, + kube::{ + core::DeserializeGuard, + runtime::{ + Controller, + events::{Recorder, Reporter}, + watcher, + }, + }, + logging::controller::report_controller_reconciled, telemetry::Tracing, utils::signal::SignalWatcher, v2::types::{kubernetes::NamespaceName, operator::ClusterName}, }; +use crate::topic::controller::{self, TOPIC_FULL_CONTROLLER_NAME}; + mod config; +mod topic; + +pub const AGENT_NAME: &str = "kafka-agent.stackable.tech"; mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); @@ -52,7 +71,11 @@ async fn main() -> anyhow::Result<()> { cluster_name, cluster_namespace, client_security, - common: CommonOptions { telemetry, .. }, + common: + CommonOptions { + telemetry, + cluster_info, + }, }) => { let _tracing_guard = Tracing::pre_configured(built_info::PKG_NAME, telemetry).init()?; @@ -67,18 +90,61 @@ async fn main() -> anyhow::Result<()> { ); let security = config::security(client_security); - let bootstrap_servers = - config::read_bootstrap_servers(Path::new(config::BOOTSTRAP_SERVERS_FILE)); tracing::info!( %cluster_name, %cluster_namespace, ?security, - ?bootstrap_servers, "Configured the agent" ); + // Watches for the SIGTERM signal and gracefully shuts down the controller. let sigterm_watcher = SignalWatcher::sigterm()?; - sigterm_watcher.handle().await; + + let client = + client::initialize_operator(Some(AGENT_NAME.to_owned()), &cluster_info).await?; + + let event_recorder = Arc::new(Recorder::new( + client.as_kube_client(), + Reporter { + controller: TOPIC_FULL_CONTROLLER_NAME.to_string(), + instance: None, + }, + )); + + let topic_controller = Controller::new( + client.get_api::>(cluster_namespace.as_ref()), + watcher::Config::default(), + ); + let topic_store = topic_controller.store(); + topic_controller + .graceful_shutdown_on(sigterm_watcher.handle()) + .run( + controller::reconcile_topic, + controller::error_policy, + Arc::new(controller::Ctx::new( + client, + cluster_name, + security, + topic_store, + )), + ) + .for_each_concurrent( + 16, // concurrency limit + move |result| { + // The event_recorder needs to be shared across all invocations, so that + // events are correctly aggregated + let event_recorder = event_recorder.clone(); + async move { + report_controller_reconciled( + &event_recorder, + TOPIC_FULL_CONTROLLER_NAME, + &result, + ) + .await; + } + }, + ) + .await; } } diff --git a/rust/agent-binary/src/topic/condition.rs b/rust/agent-binary/src/topic/condition.rs new file mode 100644 index 00000000..d29c651d --- /dev/null +++ b/rust/agent-binary/src/topic/condition.rs @@ -0,0 +1,302 @@ +//! The `Ready` and `Synced` conditions of a `KafkaTopic` for the outcome of a sync. + +use stackable_kafka_admin::client; +use stackable_operator::commons::platform_access::condition::{ + ManagedCondition, ManagedConditionType, +}; + +const MAX_LISTED_PARTITIONS: usize = 10; + +#[derive(Debug, Eq, PartialEq)] +pub enum TopicSyncOutcome { + /// The topic matches the spec + Synced { + partitions: usize, + reassigning_partitions: Vec, + }, + UnsupportedChange { + messages: Vec, + }, + ClusterNotReady, + InvalidSpec { + message: String, + }, + /// Another `KafkaTopic` manages the same topic + Conflict { + message: String, + }, + /// The topic exists in Kafka, but this `KafkaTopic` does not own it and is not allowed to adopt + /// it + AlreadyExists { + message: String, + }, + KafkaRejected { + message: String, + }, + ConnectionFailed { + message: String, + }, +} + +impl TopicSyncOutcome { + pub fn from_client_error(error: &client::Error) -> Self { + let message = error_chain(error); + if error.is_rejected_by_kafka() { + TopicSyncOutcome::KafkaRejected { message } + } else { + TopicSyncOutcome::ConnectionFailed { message } + } + } +} + +/// The conditions to update. `Ready` says whether the topic is usable through this `KafkaTopic` +/// and is only updated if that is known. +pub fn conditions(outcome: &TopicSyncOutcome) -> Vec { + let available = ready(true, "Available", "the topic exists in Kafka"); + match outcome { + TopicSyncOutcome::Synced { + partitions, + reassigning_partitions, + } => { + let synced = if reassigning_partitions.is_empty() { + synced(true, "Synced", "the topic matches the spec") + } else { + synced( + true, + "ReassignmentInProgress", + &reassignment_message(reassigning_partitions, *partitions), + ) + }; + vec![available, synced] + } + TopicSyncOutcome::UnsupportedChange { messages } => vec![ + available, + synced(false, "UnsupportedChange", &messages.join("; ")), + ], + TopicSyncOutcome::ClusterNotReady => { + let message = "the KafkaCluster has no bootstrap servers yet"; + vec![ + ready(false, "ClusterNotReady", message), + synced(false, "ClusterNotReady", message), + ] + } + TopicSyncOutcome::Conflict { message } => vec![ + ready(false, "Conflict", "another KafkaTopic manages the topic"), + synced(false, "Conflict", message), + ], + TopicSyncOutcome::AlreadyExists { message } => vec![ + ready( + false, + "AlreadyExists", + "the topic exists in Kafka, but is not owned by this KafkaTopic", + ), + synced(false, "AlreadyExists", message), + ], + TopicSyncOutcome::InvalidSpec { message } => vec![synced(false, "InvalidSpec", message)], + TopicSyncOutcome::KafkaRejected { message } => { + vec![synced(false, "KafkaRejected", message)] + } + TopicSyncOutcome::ConnectionFailed { message } => { + vec![synced(false, "ConnectionFailed", message)] + } + } +} + +/// Renders the error and its sources, skipping a source whose message the previous error already +/// contains. +fn error_chain(error: &dyn std::error::Error) -> String { + snafu::CleanedErrorText::new(error) + .map(|(_, text, _)| text) + .filter(|text| !text.is_empty()) + .collect::>() + .join(": ") +} + +fn reassignment_message(reassigning_partitions: &[i32], partitions: usize) -> String { + let listed = reassigning_partitions + .iter() + .take(MAX_LISTED_PARTITIONS) + .map(i32::to_string) + .collect::>() + .join(", "); + let more = reassigning_partitions + .len() + .saturating_sub(MAX_LISTED_PARTITIONS); + if more > 0 { + format!("partitions {listed} and {more} more of {partitions} are being reassigned") + } else { + format!("partitions {listed} of {partitions} are being reassigned") + } +} + +fn ready(status: bool, reason: &str, message: &str) -> ManagedCondition { + condition(ManagedConditionType::Ready, status, reason, message) +} + +fn synced(status: bool, reason: &str, message: &str) -> ManagedCondition { + condition(ManagedConditionType::Synced, status, reason, message) +} + +fn condition( + type_: ManagedConditionType, + status: bool, + reason: &str, + message: &str, +) -> ManagedCondition { + ManagedCondition { + type_, + status, + reason: reason.to_owned(), + message: message.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use snafu::Snafu; + + use super::*; + + fn summary(conditions: &[ManagedCondition]) -> Vec<(String, bool, &str)> { + conditions + .iter() + .map(|condition| { + ( + condition.type_.to_string(), + condition.status, + condition.reason.as_str(), + ) + }) + .collect() + } + + #[test] + fn mark_a_synced_topic() { + let conditions = conditions(&TopicSyncOutcome::Synced { + partitions: 6, + reassigning_partitions: vec![], + }); + + assert_eq!( + vec![ + ("Ready".to_owned(), true, "Available"), + ("Synced".to_owned(), true, "Synced") + ], + summary(&conditions) + ); + } + + #[test] + fn keep_a_reassigned_topic_synced() { + let conditions = conditions(&TopicSyncOutcome::Synced { + partitions: 6, + reassigning_partitions: vec![0, 2], + }); + + assert_eq!( + vec![ + ("Ready".to_owned(), true, "Available"), + ("Synced".to_owned(), true, "ReassignmentInProgress") + ], + summary(&conditions) + ); + assert_eq!( + "partitions 0, 2 of 6 are being reassigned", + conditions[1].message + ); + } + + #[test] + fn limit_the_listed_reassigning_partitions() { + let message = reassignment_message(&(0..12).collect::>(), 20); + + assert_eq!( + "partitions 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 and 2 more of 20 are being reassigned", + message + ); + } + + #[test] + fn only_update_synced_after_an_error() { + let conditions = conditions(&TopicSyncOutcome::ConnectionFailed { + message: "timed out".to_owned(), + }); + + assert_eq!( + vec![("Synced".to_owned(), false, "ConnectionFailed")], + summary(&conditions) + ); + } + + #[test] + fn mark_an_unsupported_change() { + let conditions = conditions(&TopicSyncOutcome::UnsupportedChange { + messages: vec!["first".to_owned(), "second".to_owned()], + }); + + assert_eq!( + vec![ + ("Ready".to_owned(), true, "Available"), + ("Synced".to_owned(), false, "UnsupportedChange") + ], + summary(&conditions) + ); + assert_eq!("first; second", conditions[1].message); + } + + #[test] + fn leave_ready_untouched_for_an_invalid_spec() { + let conditions = conditions(&TopicSyncOutcome::InvalidSpec { + message: "invalid".to_owned(), + }); + + assert_eq!( + vec![("Synced".to_owned(), false, "InvalidSpec")], + summary(&conditions) + ); + } + + #[test] + fn keep_the_ready_message_fixed_for_a_foreign_topic() { + let conditions = conditions(&TopicSyncOutcome::AlreadyExists { + message: "set managementPolicy.adoption to Adopt".to_owned(), + }); + + assert_eq!( + vec![ + ("Ready".to_owned(), false, "AlreadyExists"), + ("Synced".to_owned(), false, "AlreadyExists") + ], + summary(&conditions) + ); + assert_eq!( + "the topic exists in Kafka, but is not owned by this KafkaTopic", + conditions[0].message + ); + assert_eq!( + "set managementPolicy.adoption to Adopt", + conditions[1].message + ); + } + + #[derive(Debug, Snafu)] + enum Outer { + #[snafu(display("failed to connect to the bootstrap server kafka-0:9093"))] + Connect { source: Inner }, + } + + #[derive(Debug, Snafu)] + #[snafu(display("invalid peer certificate: UnknownIssuer"))] + struct Inner; + + #[test] + fn render_the_error_chain() { + let error = Outer::Connect { source: Inner }; + + assert_eq!( + "failed to connect to the bootstrap server kafka-0:9093: invalid peer certificate: \ + UnknownIssuer", + error_chain(&error) + ); + } +} diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs new file mode 100644 index 00000000..484bd9e1 --- /dev/null +++ b/rust/agent-binary/src/topic/controller.rs @@ -0,0 +1,702 @@ +//! The `KafkaTopic` controller: reconcile, cleanup on deletion and error policy. + +use std::{collections::BTreeSet, path::Path, sync::Arc, time::Duration}; + +use const_format::concatcp; +use serde_json::json; +use snafu::{ResultExt, Snafu}; +use stackable_kafka_admin::{ + bootstrap::BrokerAddress, + client::{self, KafkaAdminClient, Topic}, + connection::Security, +}; +use stackable_kafka_shared::crd::topic::v1alpha1::{ + KafkaTopic, KafkaTopicPartitionStatus, KafkaTopicStatus, +}; +use stackable_operator::{ + commons::platform_access::{ + condition::update_conditions, + ownership::{OWNED_SINCE_ANNOTATION, is_owned}, + }, + k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, + kube::{ + ResourceExt, + core::{DeserializeGuard, error_boundary}, + runtime::{controller::Action, reflector::Store}, + }, + logging::controller::ReconcilerError, + v2::types::operator::ClusterName, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; +use tokio::sync::Mutex; +use tracing::info; + +use super::{ + condition::{TopicSyncOutcome, conditions}, + managed_config_keys::{MANAGED_CONFIG_KEYS_ANNOTATION, managed_config_keys, render}, + plan::{ObservedTopic, TopicPlan, plan}, +}; +use crate::{ + AGENT_NAME, + config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}, +}; + +pub const TOPIC_CONTROLLER_NAME: &str = "kafkatopic"; +pub const TOPIC_FULL_CONTROLLER_NAME: &str = concatcp!(TOPIC_CONTROLLER_NAME, ".", AGENT_NAME); +const FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; +const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; + +/// Set by `kafka-reassign-partitions` and Cruise Control for the duration of a reassignment. +const THROTTLE_CONFIG_KEYS: [&str; 2] = [ + "leader.replication.throttled.replicas", + "follower.replication.throttled.replicas", +]; + +const RESYNC_INTERVAL: Duration = Duration::from_secs(5 * 60); +const RETRY_INTERVAL: Duration = Duration::from_secs(10); +const ERROR_RETRY_INTERVAL: Duration = Duration::from_secs(30); + +#[derive(Debug, Snafu, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("the KafkaTopic is invalid"))] + InvalidKafkaTopic { + source: error_boundary::InvalidObject, + }, + + #[snafu(display("failed to read the bootstrap servers"))] + ReadBootstrapServers { source: crate::config::Error }, + + #[snafu(display("failed to talk to Kafka"))] + Kafka { source: client::Error }, + + #[snafu(display("failed to annotate the KafkaTopic"))] + Annotate { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to update the status of the KafkaTopic"))] + UpdateStatus { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to update the finalizers of the KafkaTopic"))] + UpdateFinalizers { + source: stackable_operator::client::Error, + }, +} + +type Result = std::result::Result; + +impl ReconcilerError for Error { + fn category(&self) -> &'static str { + ErrorDiscriminants::from(self).into() + } +} + +pub struct Ctx { + client: stackable_operator::client::Client, + cluster_name: ClusterName, + security: Security, + topics: Store>, + /// One client per agent, rebuilt when the bootstrap servers change + kafka: Mutex>, +} + +impl Ctx { + /// `topics` is the store of the controller, used to detect conflicts between `KafkaTopic`s. + pub fn new( + client: stackable_operator::client::Client, + cluster_name: ClusterName, + security: Security, + topics: Store>, + ) -> Self { + Self { + client, + cluster_name, + security, + topics, + kafka: Mutex::new(None), + } + } +} + +struct CachedClient { + bootstrap_servers: Vec, + client: KafkaAdminClient, +} + +pub async fn reconcile_topic( + topic: Arc>, + ctx: Arc, +) -> Result { + info!("Starting reconcile"); + + // dereference + let topic = topic + .0 + .as_ref() + .map_err(error_boundary::InvalidObject::clone) + .context(InvalidKafkaTopicSnafu)?; + if topic.spec.cluster_name != ctx.cluster_name { + return Ok(Action::await_change()); + } + + // finalizer (Kubernetes) + if let Some(action) = reconcile_finalizer(topic, &ctx).await? { + return Ok(action); + } + + // sync (Kafka + Kubernetes) + let (outcome, observed) = match sync(topic, &ctx).await { + Ok(synced) => synced, + Err(Error::Kafka { source }) => { + let outcome = TopicSyncOutcome::from_client_error(&source); + update_status(topic, &ctx, &outcome, None).await?; + return Err(Error::Kafka { source }); + } + Err(error) => return Err(error), + }; + + // update status (Kubernetes) + update_status(topic, &ctx, &outcome, observed.as_ref()).await?; + Ok(requeue(&outcome, observed.is_some())) +} + +/// Returns `Some` if the reconcile ends here. +async fn reconcile_finalizer(topic: &KafkaTopic, ctx: &Ctx) -> Result> { + let has_finalizer = topic + .finalizers() + .iter() + .any(|finalizer| finalizer == FINALIZER); + if topic.metadata.deletion_timestamp.is_some() { + if has_finalizer { + cleanup(topic, ctx).await?; + set_finalizer(topic, ctx, false).await?; + } + return Ok(Some(Action::await_change())); + } + + let needs_finalizer = topic.spec.management_policy.deletes_on_cleanup(); + if needs_finalizer != has_finalizer { + // The patch triggers another reconcile with the updated object. + set_finalizer(topic, ctx, needs_finalizer).await?; + return Ok(Some(Action::await_change())); + } + Ok(None) +} + +fn requeue(outcome: &TopicSyncOutcome, observed: bool) -> Action { + match outcome { + TopicSyncOutcome::InvalidSpec { .. } => Action::await_change(), + // After a create or a grow, the next reconcile fills in the topic id and partitions. + TopicSyncOutcome::Synced { .. } | TopicSyncOutcome::UnsupportedChange { .. } + if !observed => + { + Action::requeue(RETRY_INTERVAL) + } + TopicSyncOutcome::ClusterNotReady => Action::requeue(RETRY_INTERVAL), + _ => Action::requeue(RESYNC_INTERVAL), + } +} + +pub fn error_policy( + _topic: Arc>, + error: &Error, + _ctx: Arc, +) -> Action { + match error { + Error::InvalidKafkaTopic { .. } => Action::await_change(), + _ => Action::requeue(ERROR_RETRY_INTERVAL), + } +} + +/// Returns the outcome and, if the `KafkaTopic` owns the topic, the described topic. After a +/// partition grow, the topic is left out, because the description is outdated. +async fn sync(topic: &KafkaTopic, ctx: &Ctx) -> Result<(TopicSyncOutcome, Option)> { + let topic_name = topic.spec.topic_name.as_ref(); + + // validate (no client required) + if let Some(outcome) = validate(topic, &ctx.topics) { + return Ok((outcome, None)); + } + + // observe (Kafka) + let Some(bootstrap_servers) = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) + .context(ReadBootstrapServersSnafu)? + else { + return Ok((TopicSyncOutcome::ClusterNotReady, None)); + }; + let mut cache = ctx.kafka.lock().await; + let kafka = client_for(&mut cache, bootstrap_servers, &ctx.security); + let observed = observe(kafka, topic_name).await?; + + // claim ownership (Kubernetes) + let claimed = if is_owned(&topic.metadata) { + false + } else { + if observed.is_some() && !topic.spec.management_policy.adopts_existing() { + let outcome = TopicSyncOutcome::AlreadyExists { + message: format!( + "the topic {topic_name:?} already exists in Kafka; set \ + managementPolicy.adoption to Adopt to take it over" + ), + }; + return Ok((outcome, None)); + } + claim_ownership(topic, ctx).await?; + true + }; + + // plan (no client required) + let recorded_keys = managed_config_keys(topic); + let topic_plan = plan(topic, observed.as_ref(), &recorded_keys); + let grows_partitions = matches!( + &topic_plan, + TopicPlan::Update(update) if update.grow_partitions_to.is_some() + ); + + // execute (Kafka + Kubernetes) + let outcome = match execute(topic, ctx, kafka, topic_plan, recorded_keys).await { + // The topic was created by someone else after it was observed. + Err(Error::Kafka { source }) if claimed && source.is_topic_already_exists() => { + release_ownership(topic, ctx).await?; + return Err(Error::Kafka { source }); + } + result => result?, + }; + + let observed = observed + .filter(|_| !grows_partitions) + .map(|observed| observed.topic); + Ok((outcome, observed)) +} + +/// Returns why the `KafkaTopic` cannot be synced, if it cannot. +fn validate( + topic: &KafkaTopic, + topics: &Store>, +) -> Option { + let throttle_keys = THROTTLE_CONFIG_KEYS + .iter() + .filter(|key| topic.spec.config.contains_key(**key)) + .copied() + .collect::>(); + if !throttle_keys.is_empty() { + return Some(TopicSyncOutcome::InvalidSpec { + message: format!( + "the config keys {throttle_keys:?} are managed by partition reassignments and \ + cannot be set" + ), + }); + } + conflicting_owner(topic, topics).map(|owner| TopicSyncOutcome::Conflict { + message: format!( + "the topic {:?} is managed by the KafkaTopic {owner:?}", + topic.spec.topic_name.as_ref() + ), + }) +} + +async fn observe(kafka: &mut KafkaAdminClient, topic_name: &str) -> Result> { + let Some(topic) = kafka.describe_topic(topic_name).await.context(KafkaSnafu)? else { + return Ok(None); + }; + let partitions = topic + .partitions + .iter() + .map(|partition| partition.index) + .collect::>(); + Ok(Some(ObservedTopic { + configs: kafka + .describe_topic_configs(topic_name) + .await + .context(KafkaSnafu)?, + reassignments: kafka + .list_partition_reassignments(topic_name, &partitions) + .await + .context(KafkaSnafu)?, + topic, + })) +} + +async fn execute( + topic: &KafkaTopic, + ctx: &Ctx, + kafka: &mut KafkaAdminClient, + topic_plan: TopicPlan, + mut recorded_keys: BTreeSet, +) -> Result { + let topic_name = topic.spec.topic_name.as_ref(); + let spec_keys = topic.spec.config.keys().cloned().collect::>(); + + let changes_configs = match &topic_plan { + TopicPlan::Create { .. } => true, + TopicPlan::Update(update) => { + !update.set_configs.is_empty() || !update.delete_configs.is_empty() + } + }; + let intended_keys = &recorded_keys | &spec_keys; + if changes_configs && intended_keys != recorded_keys { + record_managed_config_keys(topic, ctx, &intended_keys).await?; + recorded_keys = intended_keys; + } + + let outcome = match topic_plan { + TopicPlan::Create { + partitions, + replication_factor, + configs, + } => { + kafka + .create_topic(topic_name, partitions, replication_factor, &configs) + .await + .context(KafkaSnafu)?; + info!( + topic = topic_name, + partitions, + replication_factor, + ?configs, + kafka_topic = %topic.name_any(), + "Created the topic" + ); + TopicSyncOutcome::Synced { + partitions: partitions as usize, + reassigning_partitions: Vec::new(), + } + } + TopicPlan::Update(update) => { + if let Some(partitions) = update.grow_partitions_to { + kafka + .create_partitions(topic_name, partitions) + .await + .context(KafkaSnafu)?; + info!( + topic = topic_name, + partitions, + kafka_topic = %topic.name_any(), + "Increased the number of partitions" + ); + } + if changes_configs { + kafka + .incremental_alter_configs( + topic_name, + &update.set_configs, + &update.delete_configs, + ) + .await + .context(KafkaSnafu)?; + info!( + topic = topic_name, + set = ?update.set_configs, + reset = ?update.delete_configs, + kafka_topic = %topic.name_any(), + "Changed the topic configs" + ); + } + if update.unsupported.is_empty() { + TopicSyncOutcome::Synced { + partitions: topic.spec.partitions as usize, + reassigning_partitions: update.reassigning_partitions, + } + } else { + TopicSyncOutcome::UnsupportedChange { + messages: update.unsupported, + } + } + } + }; + + if recorded_keys != spec_keys { + record_managed_config_keys(topic, ctx, &spec_keys).await?; + } + + Ok(outcome) +} + +async fn cleanup(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + if topic.spec.management_policy.deletes_on_cleanup() + && is_owned(&topic.metadata) + && conflicting_owner(topic, &ctx.topics).is_none() + { + let bootstrap_servers = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) + .context(ReadBootstrapServersSnafu)? + .unwrap_or_default(); + let mut cache = ctx.kafka.lock().await; + client_for(&mut cache, bootstrap_servers, &ctx.security) + .delete_topic(topic.spec.topic_name.as_ref()) + .await + .context(KafkaSnafu)?; + info!( + topic = topic.spec.topic_name.as_ref(), + kafka_topic = %topic.name_any(), + "Deleted the topic" + ); + } + Ok(()) +} + +/// Fails if the `KafkaTopic` was changed since it was read, so concurrent changes to the +/// finalizers are not lost. +async fn set_finalizer(topic: &KafkaTopic, ctx: &Ctx, present: bool) -> Result<()> { + let mut finalizers = topic + .finalizers() + .iter() + .filter(|finalizer| *finalizer != FINALIZER) + .cloned() + .collect::>(); + if present { + finalizers.push(FINALIZER.to_owned()); + } + let patch = json!({ + "metadata": { + "resourceVersion": topic.metadata.resource_version, + "finalizers": finalizers, + } + }); + ctx.client + .merge_patch(topic, patch) + .await + .context(UpdateFinalizersSnafu)?; + Ok(()) +} + +/// Returns the cached client, rebuilt if the bootstrap servers changed. +fn client_for<'a>( + cache: &'a mut Option, + bootstrap_servers: Vec, + security: &Security, +) -> &'a mut KafkaAdminClient { + if cache + .as_ref() + .is_some_and(|cached| cached.bootstrap_servers != bootstrap_servers) + { + *cache = None; + } + let cached = cache.get_or_insert_with(|| { + info!(?bootstrap_servers, "Connecting to new bootstrap servers"); + CachedClient { + bootstrap_servers: bootstrap_servers.clone(), + client: KafkaAdminClient::new(bootstrap_servers, security.clone(), KAFKA_CLIENT_ID), + } + }); + &mut cached.client +} + +/// The oldest other `KafkaTopic` for the same topic of the same cluster, by creation time and +/// name. Checked against all `KafkaTopic`s in the store, so the result does not depend on the +/// reconcile order. +fn conflicting_owner( + topic: &KafkaTopic, + topics: &Store>, +) -> Option { + let key = |topic: &KafkaTopic| (topic.metadata.creation_timestamp.clone(), topic.name_any()); + topics + .state() + .iter() + .filter_map(|other| other.0.as_ref().ok()) + .filter(|other| { + other.spec.cluster_name == topic.spec.cluster_name + && other.spec.topic_name == topic.spec.topic_name + }) + .min_by_key(|other| key(other)) + .filter(|oldest| key(oldest) != key(topic)) + .map(ResourceExt::name_any) +} + +/// Fails if the `KafkaTopic` was changed since it was read, so ownership is never claimed twice. +async fn claim_ownership(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + let patch = json!({ + "metadata": { + "resourceVersion": topic.metadata.resource_version, + "annotations": { OWNED_SINCE_ANNOTATION: Timestamp::now().to_string() }, + } + }); + ctx.client + .merge_patch(topic, patch) + .await + .context(AnnotateSnafu)?; + Ok(()) +} + +async fn release_ownership(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + let patch = json!({ + "metadata": { + "annotations": { + OWNED_SINCE_ANNOTATION: null, + MANAGED_CONFIG_KEYS_ANNOTATION: null, + }, + } + }); + ctx.client + .merge_patch(topic, patch) + .await + .context(AnnotateSnafu)?; + Ok(()) +} + +async fn record_managed_config_keys( + topic: &KafkaTopic, + ctx: &Ctx, + keys: &BTreeSet, +) -> Result<()> { + let patch = json!({ + "metadata": { "annotations": { MANAGED_CONFIG_KEYS_ANNOTATION: render(keys) } } + }); + ctx.client + .merge_patch(topic, patch) + .await + .context(AnnotateSnafu)?; + Ok(()) +} + +/// Patches the status only if it changed, so the agent does not trigger its own watch. +async fn update_status( + topic: &KafkaTopic, + ctx: &Ctx, + outcome: &TopicSyncOutcome, + observed: Option<&Topic>, +) -> Result<()> { + let current = topic.status.clone().unwrap_or_default(); + let mut status = KafkaTopicStatus { + conditions: update_conditions( + ¤t.conditions, + conditions(outcome), + &Time(Timestamp::now()), + ), + ..current.clone() + }; + match (outcome, observed) { + (_, Some(observed)) => { + status.topic_id = Some(observed.id.clone()); + status.partitions = observed + .partitions + .iter() + .map(|partition| KafkaTopicPartitionStatus { + index: partition.index, + replicas: partition.replicas.clone(), + }) + .collect(); + } + (TopicSyncOutcome::AlreadyExists { .. } | TopicSyncOutcome::Conflict { .. }, None) => { + status.topic_id = None; + status.partitions = Vec::new(); + } + _ => {} + } + + if status != current { + ctx.client + .apply_patch_status(AGENT_NAME, topic, &status) + .await + .context(UpdateStatusSnafu)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use stackable_operator::{ + k8s_openapi::apimachinery::pkg::apis::meta::v1::Time, kube::runtime::watcher, + }; + + use super::*; + use crate::topic::test_support::kafka_topic; + + fn created_topic( + name: &str, + created_at: i64, + topic_name: &str, + ) -> DeserializeGuard { + let mut topic = kafka_topic(name, topic_name); + topic.metadata.creation_timestamp = Some(Time( + Timestamp::from_second(created_at).expect("the timestamp should be valid"), + )); + DeserializeGuard(Ok(topic)) + } + + fn store(topics: Vec>) -> Store> { + let (store, mut writer) = stackable_operator::kube::runtime::reflector::store(); + for topic in topics { + writer.apply_watcher_event(&watcher::Event::Apply(topic)); + } + store + } + + fn inner(topic: &DeserializeGuard) -> &KafkaTopic { + topic.0.as_ref().expect("the topic should be valid") + } + + #[test] + fn the_oldest_kafka_topic_owns_the_topic() { + let older = created_topic("older", 10, "orders"); + let newer = created_topic("newer", 20, "orders"); + let other_topic = created_topic("other", 5, "payments"); + let topics = store(vec![older.clone(), newer.clone(), other_topic]); + + assert_eq!(None, conflicting_owner(inner(&older), &topics)); + assert_eq!( + Some("older".to_owned()), + conflicting_owner(inner(&newer), &topics) + ); + } + + #[test] + fn the_name_decides_between_kafka_topics_of_the_same_age() { + let first = created_topic("a", 10, "orders"); + let second = created_topic("b", 10, "orders"); + let topics = store(vec![second.clone(), first.clone()]); + + assert_eq!(None, conflicting_owner(inner(&first), &topics)); + assert_eq!( + Some("a".to_owned()), + conflicting_owner(inner(&second), &topics) + ); + } + + #[rstest::rstest] + #[case::no_throttle(&[("retention.ms", "1000")], false)] + #[case::throttle(&[("follower.replication.throttled.replicas", "*")], true)] + fn reject_throttle_config_keys(#[case] config: &[(&str, &str)], #[case] invalid: bool) { + let mut topic = kafka_topic("orders", "orders"); + topic.spec.config = config + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + + assert_eq!( + invalid, + matches!( + validate(&topic, &store(vec![])), + Some(TopicSyncOutcome::InvalidSpec { .. }) + ) + ); + } + + fn synced() -> TopicSyncOutcome { + TopicSyncOutcome::Synced { + partitions: 1, + reassigning_partitions: vec![], + } + } + + #[rstest::rstest] + #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: String::new() }, false, Action::await_change())] + #[case::created(synced(), false, Action::requeue(RETRY_INTERVAL))] + #[case::cluster_not_ready( + TopicSyncOutcome::ClusterNotReady, + false, + Action::requeue(RETRY_INTERVAL) + )] + #[case::synced(synced(), true, Action::requeue(RESYNC_INTERVAL))] + #[case::grown_with_unsupported_change(TopicSyncOutcome::UnsupportedChange { messages: vec![] }, false, Action::requeue(RETRY_INTERVAL))] + #[case::unsupported_change(TopicSyncOutcome::UnsupportedChange { messages: vec![] }, true, Action::requeue(RESYNC_INTERVAL))] + fn requeue_by_outcome( + #[case] outcome: TopicSyncOutcome, + #[case] observed: bool, + #[case] expected: Action, + ) { + assert_eq!(expected, requeue(&outcome, observed)); + } +} diff --git a/rust/agent-binary/src/topic/managed_config_keys.rs b/rust/agent-binary/src/topic/managed_config_keys.rs new file mode 100644 index 00000000..069d3230 --- /dev/null +++ b/rust/agent-binary/src/topic/managed_config_keys.rs @@ -0,0 +1,69 @@ +//! The topic config keys the agent manages, recorded on the `KafkaTopic`. +//! +//! Keys removed from the spec are reset in Kafka, but only if the agent set them before. Kafka +//! only accepts its known topic config names, none of which contain a comma, so the keys are +//! stored comma-separated. + +use std::collections::BTreeSet; + +use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; +use stackable_operator::kube::ResourceExt; + +pub const MANAGED_CONFIG_KEYS_ANNOTATION: &str = "kafka.stackable.tech/managed-config-keys"; + +pub fn managed_config_keys(topic: &KafkaTopic) -> BTreeSet { + topic + .annotations() + .get(MANAGED_CONFIG_KEYS_ANNOTATION) + .map(|value| { + value + .split(',') + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default() +} + +pub fn render(keys: &BTreeSet) -> String { + keys.iter() + .map(String::as_str) + .collect::>() + .join(",") +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use rstest::rstest; + + use super::*; + use crate::topic::test_support::kafka_topic; + + #[rstest] + #[case::no_annotation(None, &[])] + #[case::empty(Some(""), &[])] + #[case::keys(Some("retention.ms,cleanup.policy"), &["cleanup.policy", "retention.ms"])] + #[case::spaces_and_empty_entries(Some(" retention.ms, ,"), &["retention.ms"])] + fn read_managed_config_keys(#[case] annotation: Option<&str>, #[case] expected: &[&str]) { + let mut topic = kafka_topic("orders", "orders"); + topic.metadata.annotations = annotation.map(|value| { + BTreeMap::from([(MANAGED_CONFIG_KEYS_ANNOTATION.to_owned(), value.to_owned())]) + }); + + let expected = expected + .iter() + .map(|key| key.to_string()) + .collect::>(); + assert_eq!(expected, managed_config_keys(&topic)); + } + + #[test] + fn render_sorted_keys() { + let keys = BTreeSet::from(["retention.ms".to_owned(), "cleanup.policy".to_owned()]); + + assert_eq!("cleanup.policy,retention.ms", render(&keys)); + } +} diff --git a/rust/agent-binary/src/topic/mod.rs b/rust/agent-binary/src/topic/mod.rs new file mode 100644 index 00000000..a1ef9ffd --- /dev/null +++ b/rust/agent-binary/src/topic/mod.rs @@ -0,0 +1,8 @@ +//! Reconciles `KafkaTopic`s into topics in Kafka. + +mod condition; +pub mod controller; +mod managed_config_keys; +mod plan; +#[cfg(test)] +mod test_support; diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs new file mode 100644 index 00000000..44e5f71f --- /dev/null +++ b/rust/agent-binary/src/topic/plan.rs @@ -0,0 +1,324 @@ +//! What the agent has to do in Kafka to bring a topic to the state of its `KafkaTopic` spec. + +use std::collections::{BTreeMap, BTreeSet}; + +use stackable_kafka_admin::client::{PartitionReassignment, Topic}; +use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; + +/// The state of an existing topic in Kafka. +pub struct ObservedTopic { + pub topic: Topic, + pub configs: BTreeMap, + pub reassignments: Vec, +} + +#[derive(Debug, Eq, PartialEq)] +pub enum TopicPlan { + Create { + partitions: i32, + replication_factor: i16, + configs: BTreeMap, + }, + Update(TopicUpdate), +} + +/// Changes to an existing topic. No grow, configs or unsupported differences means the topic +/// matches the spec. +#[derive(Debug, Default, Eq, PartialEq)] +pub struct TopicUpdate { + pub grow_partitions_to: Option, + pub set_configs: BTreeMap, + pub delete_configs: BTreeSet, + /// Differences the agent does not apply, as messages for the status + pub unsupported: Vec, + pub reassigning_partitions: Vec, +} + +pub fn plan( + topic: &KafkaTopic, + observed: Option<&ObservedTopic>, + managed_config_keys: &BTreeSet, +) -> TopicPlan { + let spec = &topic.spec; + let Some(observed) = observed else { + return TopicPlan::Create { + partitions: spec.partitions, + replication_factor: spec.replication_factor, + configs: spec.config.clone(), + }; + }; + + let mut update = TopicUpdate::default(); + + let observed_partitions = observed.topic.partitions.len() as i32; + if spec.partitions > observed_partitions { + update.grow_partitions_to = Some(spec.partitions); + } else if spec.partitions < observed_partitions { + update.unsupported.push(format!( + "the topic has {observed_partitions} partitions, but the spec requests {}; partitions cannot be \ + removed", + spec.partitions + )); + } + + if let Some(message) = replication_factor_difference( + spec.replication_factor, + &target_replication_factors(observed), + ) { + update.unsupported.push(message); + } + + update.set_configs = spec + .config + .iter() + .filter(|(key, value)| observed.configs.get(*key) != Some(*value)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + update.delete_configs = managed_config_keys + .iter() + .filter(|key| !spec.config.contains_key(*key) && observed.configs.contains_key(*key)) + .cloned() + .collect(); + + update.reassigning_partitions = observed + .reassignments + .iter() + .map(|reassignment| reassignment.index) + .collect(); + + TopicPlan::Update(update) +} + +/// The replication factor of each partition, excluding replicas a reassignment removes. +fn target_replication_factors(observed: &ObservedTopic) -> BTreeSet { + observed + .topic + .partitions + .iter() + .map(|partition| { + let removing = observed + .reassignments + .iter() + .find(|reassignment| reassignment.index == partition.index) + .map_or(0, |reassignment| reassignment.removing_replicas.len()); + partition.replicas.len().saturating_sub(removing) + }) + .collect() +} + +fn replication_factor_difference( + desired: i16, + replication_factors: &BTreeSet, +) -> Option { + let (Some(min), Some(max)) = (replication_factors.first(), replication_factors.last()) else { + return None; + }; + + let hint = "change it with kafka-reassign-partitions"; + if min != max { + Some(format!( + "the partitions have between {min} and {max} replicas, but the spec requests a \ + replication factor of {desired}; {hint}" + )) + } else if *min != desired as usize { + Some(format!( + "the topic has a replication factor of {min}, but the spec requests {desired}; {hint}" + )) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use stackable_kafka_admin::client::Partition; + + use super::*; + use crate::topic::test_support::kafka_topic; + + fn topic(partitions: i32, replication_factor: i16, config: &[(&str, &str)]) -> KafkaTopic { + let mut topic = kafka_topic("orders", "orders"); + topic.spec.partitions = partitions; + topic.spec.replication_factor = replication_factor; + topic.spec.config = configs(config); + topic + } + + fn configs(entries: &[(&str, &str)]) -> BTreeMap { + entries + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect() + } + + fn keys(keys: &[&str]) -> BTreeSet { + keys.iter().map(|key| key.to_string()).collect() + } + + /// A topic whose partitions have the given replicas + fn observed(replicas: &[&[i32]], config: &[(&str, &str)]) -> ObservedTopic { + ObservedTopic { + topic: Topic { + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partitions: replicas + .iter() + .zip(0..) + .map(|(replicas, index)| Partition { + index, + replicas: replicas.to_vec(), + }) + .collect(), + }, + configs: configs(config), + reassignments: vec![], + } + } + + #[test] + fn create_missing_topic() { + let plan = plan(&topic(3, 2, &[("retention.ms", "1000")]), None, &keys(&[])); + + assert_eq!( + TopicPlan::Create { + partitions: 3, + replication_factor: 2, + configs: configs(&[("retention.ms", "1000")]), + }, + plan + ); + } + + #[test] + fn do_nothing_for_a_matching_topic() { + let observed = observed(&[&[1, 2], &[2, 3]], &[("retention.ms", "1000")]); + + let plan = plan( + &topic(2, 2, &[("retention.ms", "1000")]), + Some(&observed), + &keys(&["retention.ms"]), + ); + + assert_eq!(TopicPlan::Update(TopicUpdate::default()), plan); + } + + #[test] + fn grow_partitions() { + let observed = observed(&[&[1], &[2]], &[]); + + let plan = plan(&topic(4, 1, &[]), Some(&observed), &keys(&[])); + + assert_eq!( + TopicPlan::Update(TopicUpdate { + grow_partitions_to: Some(4), + ..TopicUpdate::default() + }), + plan + ); + } + + #[test] + fn report_fewer_partitions_as_unsupported() { + let observed = observed(&[&[1], &[2], &[3]], &[]); + + let TopicPlan::Update(update) = plan(&topic(2, 1, &[]), Some(&observed), &keys(&[])) else { + panic!("the topic exists"); + }; + + assert_eq!(None, update.grow_partitions_to); + assert_eq!(1, update.unsupported.len()); + assert!(update.unsupported[0].contains("cannot be removed")); + } + + #[test] + fn set_changed_and_reset_removed_configs() { + let observed = observed( + &[&[1]], + &[ + ("retention.ms", "1000"), + ("cleanup.policy", "compact"), + ("segment.bytes", "1024"), + ("leader.replication.throttled.replicas", "0:1"), + ], + ); + + let plan = plan( + &topic( + 1, + 1, + &[("retention.ms", "2000"), ("max.message.bytes", "10")], + ), + Some(&observed), + &keys(&["retention.ms", "cleanup.policy", "min.insync.replicas"]), + ); + + assert_eq!( + TopicPlan::Update(TopicUpdate { + set_configs: configs(&[("max.message.bytes", "10"), ("retention.ms", "2000")]), + // `segment.bytes` and the throttle were set by others, `min.insync.replicas` is + // already gone + delete_configs: keys(&["cleanup.policy"]), + ..TopicUpdate::default() + }), + plan + ); + } + + #[test] + fn report_a_different_replication_factor_as_unsupported() { + let observed = observed(&[&[1, 2, 3], &[2, 3, 4]], &[]); + + let TopicPlan::Update(update) = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])) else { + panic!("the topic exists"); + }; + + assert_eq!(1, update.unsupported.len()); + assert!(update.unsupported[0].contains("replication factor of 3")); + } + + #[test] + fn report_uneven_replica_counts_as_unsupported() { + let observed = observed(&[&[1, 2], &[2, 3, 4]], &[]); + + let TopicPlan::Update(update) = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])) else { + panic!("the topic exists"); + }; + + assert_eq!(1, update.unsupported.len()); + assert!(update.unsupported[0].contains("between 2 and 3 replicas")); + } + + #[test] + fn accept_a_reassignment_towards_the_desired_replication_factor() { + let mut observed = observed(&[&[4, 5, 1, 2], &[2, 3]], &[]); + observed.reassignments = vec![PartitionReassignment { + index: 0, + adding_replicas: vec![4, 5], + removing_replicas: vec![1, 2], + }]; + + let plan = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])); + + assert_eq!( + TopicPlan::Update(TopicUpdate { + reassigning_partitions: vec![0], + ..TopicUpdate::default() + }), + plan + ); + } + + #[test] + fn exclude_removed_replicas_from_the_replication_factor() { + let mut observed = observed(&[&[4, 5, 1, 2], &[2, 3, 4]], &[]); + observed.reassignments = vec![PartitionReassignment { + index: 0, + adding_replicas: vec![4, 5], + removing_replicas: vec![1, 2], + }]; + + assert_eq!( + BTreeSet::from([2, 3]), + target_replication_factors(&observed) + ); + } +} diff --git a/rust/agent-binary/src/topic/test_support.rs b/rust/agent-binary/src/topic/test_support.rs new file mode 100644 index 00000000..44830af9 --- /dev/null +++ b/rust/agent-binary/src/topic/test_support.rs @@ -0,0 +1,23 @@ +use std::str::FromStr; + +use stackable_kafka_shared::crd::topic::{ + TopicName, + v1alpha1::{KafkaTopic, KafkaTopicSpec}, +}; +use stackable_operator::v2::types::operator::ClusterName; + +/// A `KafkaTopic` of the cluster `simple-kafka` with one partition, one replica and no configs. +pub fn kafka_topic(name: &str, topic_name: &str) -> KafkaTopic { + KafkaTopic::new( + name, + KafkaTopicSpec { + cluster_name: ClusterName::from_str("simple-kafka") + .expect("the cluster name should be valid"), + management_policy: Default::default(), + topic_name: TopicName::from_str(topic_name).expect("the topic name should be valid"), + partitions: 1, + replication_factor: 1, + config: Default::default(), + }, + ) +} diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 7b855a8e..c919c3cd 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -82,6 +82,24 @@ pub mod versioned { /// `Ready` and `Synced`. #[serde(default)] pub conditions: Vec, + + /// The topic id in Kafka, as shown by `kafka-topics.sh`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_id: Option, + + /// The partitions of the topic in Kafka. + #[serde(default)] + pub partitions: Vec, + } + + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaTopicPartitionStatus { + pub index: i32, + + /// Broker ids holding a replica, the preferred leader first. During a reassignment, this + /// includes the replicas being added and removed. + pub replicas: Vec, } } From 81a49a1dd3ebbb93fdd737d19c6704d876afea63 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 23 Oct 2026 14:36:23 +0200 Subject: [PATCH 08/33] wip: add kuttl tests --- Cargo.nix | 18 ++--- crate-hashes.json | 18 ++--- .../topic-authentication/00-patch-ns.yaml.j2 | 9 +++ .../05-create-agent-credential.yaml.j2 | 29 +++++++ .../kuttl/topic-authentication/10-assert.yaml | 20 +++++ .../10-install-kafka.yaml | 7 ++ .../topic-authentication/10_kafka.yaml.j2 | 77 +++++++++++++++++++ .../topic-authentication/11-assert.yaml.j2 | 14 ++++ .../kuttl/topic-authentication/20-assert.yaml | 8 ++ .../topic-authentication/20-create-topic.yaml | 10 +++ .../kuttl/topic-authentication/kafka-cli.sh | 13 ++++ .../topic-authentication/topic-condition.sh | 11 +++ .../templates/kuttl/topic/00-patch-ns.yaml.j2 | 9 +++ tests/templates/kuttl/topic/10-assert.yaml | 42 ++++++++++ .../kuttl/topic/10-install-kafka.yaml.j2 | 34 ++++++++ tests/templates/kuttl/topic/20-assert.yaml | 12 +++ .../kuttl/topic/20-create-topic.yaml | 13 ++++ tests/templates/kuttl/topic/30-assert.yaml | 9 +++ .../kuttl/topic/30-change-topic.yaml | 13 ++++ .../topic/40-set-config-out-of-band.yaml | 6 ++ tests/templates/kuttl/topic/41-assert.yaml | 12 +++ .../kuttl/topic/41-remove-config-key.yaml | 8 ++ tests/templates/kuttl/topic/50-assert.yaml | 9 +++ .../kuttl/topic/50-unsupported-change.yaml | 13 ++++ tests/templates/kuttl/topic/51-assert.yaml | 6 ++ .../topic/51-revert-unsupported-change.yaml | 12 +++ tests/templates/kuttl/topic/60-assert.yaml | 11 +++ .../kuttl/topic/60-invalid-spec.yaml | 13 ++++ .../kuttl/topic/70-create-foreign-topic.yaml | 6 ++ tests/templates/kuttl/topic/71-assert.yaml | 8 ++ .../kuttl/topic/71-refuse-foreign-topic.yaml | 10 +++ .../kuttl/topic/72-adopt-foreign-topic.yaml | 12 +++ tests/templates/kuttl/topic/72-assert.yaml | 8 ++ tests/templates/kuttl/topic/80-assert.yaml | 9 +++ .../topic/80-create-conflicting-topic.yaml | 11 +++ tests/templates/kuttl/topic/81-assert.yaml | 6 ++ .../topic/81-delete-conflicting-topic.yaml | 7 ++ tests/templates/kuttl/topic/90-assert.yaml | 8 ++ .../90-delete-legacy-topic-in-kafka.yaml | 13 ++++ tests/templates/kuttl/topic/91-assert.yaml | 11 +++ .../kuttl/topic/91-delete-topics.yaml | 13 ++++ .../topic/95-disable-platform-access.yaml.j2 | 34 ++++++++ tests/templates/kuttl/topic/95-errors.yaml | 16 ++++ tests/templates/kuttl/topic/kafka-cli.sh | 13 ++++ .../templates/kuttl/topic/topic-condition.sh | 11 +++ tests/test-definition.yaml | 17 +++- 46 files changed, 650 insertions(+), 19 deletions(-) create mode 100644 tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 create mode 100644 tests/templates/kuttl/topic-authentication/10-assert.yaml create mode 100644 tests/templates/kuttl/topic-authentication/10-install-kafka.yaml create mode 100644 tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 create mode 100644 tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 create mode 100644 tests/templates/kuttl/topic-authentication/20-assert.yaml create mode 100644 tests/templates/kuttl/topic-authentication/20-create-topic.yaml create mode 100644 tests/templates/kuttl/topic-authentication/kafka-cli.sh create mode 100644 tests/templates/kuttl/topic-authentication/topic-condition.sh create mode 100644 tests/templates/kuttl/topic/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/topic/10-assert.yaml create mode 100644 tests/templates/kuttl/topic/10-install-kafka.yaml.j2 create mode 100644 tests/templates/kuttl/topic/20-assert.yaml create mode 100644 tests/templates/kuttl/topic/20-create-topic.yaml create mode 100644 tests/templates/kuttl/topic/30-assert.yaml create mode 100644 tests/templates/kuttl/topic/30-change-topic.yaml create mode 100644 tests/templates/kuttl/topic/40-set-config-out-of-band.yaml create mode 100644 tests/templates/kuttl/topic/41-assert.yaml create mode 100644 tests/templates/kuttl/topic/41-remove-config-key.yaml create mode 100644 tests/templates/kuttl/topic/50-assert.yaml create mode 100644 tests/templates/kuttl/topic/50-unsupported-change.yaml create mode 100644 tests/templates/kuttl/topic/51-assert.yaml create mode 100644 tests/templates/kuttl/topic/51-revert-unsupported-change.yaml create mode 100644 tests/templates/kuttl/topic/60-assert.yaml create mode 100644 tests/templates/kuttl/topic/60-invalid-spec.yaml create mode 100644 tests/templates/kuttl/topic/70-create-foreign-topic.yaml create mode 100644 tests/templates/kuttl/topic/71-assert.yaml create mode 100644 tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml create mode 100644 tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml create mode 100644 tests/templates/kuttl/topic/72-assert.yaml create mode 100644 tests/templates/kuttl/topic/80-assert.yaml create mode 100644 tests/templates/kuttl/topic/80-create-conflicting-topic.yaml create mode 100644 tests/templates/kuttl/topic/81-assert.yaml create mode 100644 tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml create mode 100644 tests/templates/kuttl/topic/90-assert.yaml create mode 100644 tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml create mode 100644 tests/templates/kuttl/topic/91-assert.yaml create mode 100644 tests/templates/kuttl/topic/91-delete-topics.yaml create mode 100644 tests/templates/kuttl/topic/95-disable-platform-access.yaml.j2 create mode 100644 tests/templates/kuttl/topic/95-errors.yaml create mode 100644 tests/templates/kuttl/topic/kafka-cli.sh create mode 100644 tests/templates/kuttl/topic/topic-condition.sh diff --git a/Cargo.nix b/Cargo.nix index 3abdc326..dcd03807 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5510,7 +5510,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "k8s_version"; authors = [ @@ -10508,7 +10508,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_certs"; authors = [ @@ -10910,7 +10910,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_operator"; authors = [ @@ -11109,7 +11109,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -11144,7 +11144,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_shared"; authors = [ @@ -11225,7 +11225,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_telemetry"; authors = [ @@ -11335,7 +11335,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_versioned"; authors = [ @@ -11385,7 +11385,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11449,7 +11449,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index e4f5f7b1..b5d1f483 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,11 +1,11 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.3": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.119.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.1.2": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.11.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.11.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.3": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.119.0": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.1.2": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.11.1": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.11.1": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd" } \ No newline at end of file diff --git a/tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 b/tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 b/tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 new file mode 100644 index 00000000..40c0d2fd --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 @@ -0,0 +1,29 @@ +{% if test_scenario['values']['platform-access-authentication'] == 'secret' %} +--- +# A CA trusted by the brokers for client authentication, and a certificate for the agent signed by +# it, as a user would provide them +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + set -eu + dir=$(mktemp -d) + printf "basicConstraints=CA:FALSE\nextendedKeyUsage=clientAuth\n" > "$dir/extensions.cnf" + openssl req -x509 -newkey rsa:2048 -nodes -days 30 \ + -subj "/CN=Test client CA" \ + -addext "basicConstraints=critical,CA:TRUE" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -keyout "$dir/ca.key" -out "$dir/ca.crt" + openssl req -newkey rsa:2048 -nodes \ + -subj "/CN=kafka-agent" \ + -keyout "$dir/tls.key" -out "$dir/tls.csr" + openssl x509 -req -days 30 -in "$dir/tls.csr" \ + -CA "$dir/ca.crt" -CAkey "$dir/ca.key" -CAcreateserial \ + -extfile "$dir/extensions.cnf" \ + -out "$dir/tls.crt" + kubectl create secret generic client-auth-ca --namespace "$NAMESPACE" \ + --from-file=ca.crt="$dir/ca.crt" --from-file=ca.key="$dir/ca.key" + kubectl create secret tls kafka-agent-credential --namespace "$NAMESPACE" \ + --cert="$dir/tls.crt" --key="$dir/tls.key" + rm -r "$dir" +{% endif %} diff --git a/tests/templates/kuttl/topic-authentication/10-assert.yaml b/tests/templates/kuttl/topic-authentication/10-assert.yaml new file mode 100644 index 00000000..5b1be639 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/10-assert.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-kafka-agent +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/topic-authentication/10-install-kafka.yaml b/tests/templates/kuttl/topic-authentication/10-install-kafka.yaml new file mode 100644 index 00000000..4652b959 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/10-install-kafka.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: > + envsubst '$NAMESPACE' < 10_kafka.yaml | + kubectl apply --namespace $NAMESPACE -f - diff --git a/tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 b/tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 new file mode 100644 index 00000000..e49e34ee --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 @@ -0,0 +1,77 @@ +{% set authentication = test_scenario['values']['platform-access-authentication'] %} +{% if authentication != 'anonymous' %} +--- +# SecretClasses and AuthenticationClasses are cluster-scoped, so the namespace keeps their names +# unique between parallel test runs. +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: client-auth-$NAMESPACE +spec: + backend: + autoTls: + ca: + secret: + name: client-auth-ca + namespace: $NAMESPACE +{% if authentication == 'secret' %} + autoGenerate: false +{% else %} + autoGenerate: true +{% endif %} +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: client-auth-$NAMESPACE +spec: + provider: + tls: + clientCertSecretClass: client-auth-$NAMESPACE +{% endif %} +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + image: +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls +{% if authentication != 'anonymous' %} + authentication: + - authenticationClass: client-auth-$NAMESPACE +{% endif %} + platformAccess: + enabled: true + authentication: +{% if authentication == 'anonymous' %} + anonymous: {} +{% elif authentication == 'secret-class' %} + tls: + secretClass: client-auth-$NAMESPACE +{% else %} + tls: + secret: kafka-agent-credential +{% endif %} + controllers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 + brokers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 b/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 new file mode 100644 index 00000000..41267d43 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 @@ -0,0 +1,14 @@ +{% set authentication = test_scenario['values']['platform-access-authentication'] %} +--- +# Runs after the agent is ready, so the commands are not polled during the startup. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +commands: +{% if authentication == 'anonymous' %} + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "discovery tls-kafka" + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_CLIENT_SECURITY")].value}')" = tls +{% else %} + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "discovery tls-kafka tls-platform-access" + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_CLIENT_SECURITY")].value}')" = mutual-tls +{% endif %} diff --git a/tests/templates/kuttl/topic-authentication/20-assert.yaml b/tests/templates/kuttl/topic-authentication/20-assert.yaml new file mode 100644 index 00000000..fcdd3b41 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/20-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Ready True Available + - script: bash topic-condition.sh orders Synced True Synced + - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 3"' diff --git a/tests/templates/kuttl/topic-authentication/20-create-topic.yaml b/tests/templates/kuttl/topic-authentication/20-create-topic.yaml new file mode 100644 index 00000000..a0020413 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/20-create-topic.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterName: test-kafka + topicName: orders + partitions: 3 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic-authentication/kafka-cli.sh b/tests/templates/kuttl/topic-authentication/kafka-cli.sh new file mode 100644 index 00000000..31627437 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/kafka-cli.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Usage: kafka-cli.sh , e.g. kafka-cli.sh kafka-topics.sh --describe --topic orders +# Runs a Kafka CLI tool in the broker Pod against the brokers. +set -euo pipefail + +# The script is expanded in the broker Pod, not here. +# shellcheck disable=SC2016 +kubectl exec --namespace "$NAMESPACE" test-kafka-broker-default-0 --container kafka -- bash -c ' + bootstrap="$(cat /stackable/listener-broker/default-address/address):$(cat /stackable/listener-broker/default-address/ports/kafka-tls)" + tool=$1 + shift + "/stackable/kafka/bin/$tool" --bootstrap-server "$bootstrap" --command-config /stackable/config/client.properties "$@" +' kafka-cli "$@" diff --git a/tests/templates/kuttl/topic-authentication/topic-condition.sh b/tests/templates/kuttl/topic-authentication/topic-condition.sh new file mode 100644 index 00000000..59a05a0d --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/topic-condition.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Usage: topic-condition.sh +# Fails unless the condition of the KafkaTopic has the given status and reason. +set -euo pipefail + +actual=$(kubectl get kafkatopic --namespace "$NAMESPACE" "$1" \ + --output "jsonpath={.status.conditions[?(@.type==\"$2\")].status}/{.status.conditions[?(@.type==\"$2\")].reason}") +if [ "$actual" != "$3/$4" ]; then + echo "$1: expected the condition $2 to be $3/$4, but it is \"$actual\"" + exit 1 +fi diff --git a/tests/templates/kuttl/topic/00-patch-ns.yaml.j2 b/tests/templates/kuttl/topic/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/topic/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/topic/10-assert.yaml b/tests/templates/kuttl/topic/10-assert.yaml new file mode 100644 index 00000000..dcb6e261 --- /dev/null +++ b/tests/templates/kuttl/topic/10-assert.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-controller-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-kafka-agent +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-kafka-agent +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-kafka-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kafka-agent-clusterrole diff --git a/tests/templates/kuttl/topic/10-install-kafka.yaml.j2 b/tests/templates/kuttl/topic/10-install-kafka.yaml.j2 new file mode 100644 index 00000000..feec661a --- /dev/null +++ b/tests/templates/kuttl/topic/10-install-kafka.yaml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + image: +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls + platformAccess: + enabled: true + authentication: + anonymous: {} + controllers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 + brokers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/topic/20-assert.yaml b/tests/templates/kuttl/topic/20-assert.yaml new file mode 100644 index 00000000..fcc58d2c --- /dev/null +++ b/tests/templates/kuttl/topic/20-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Ready True Available + - script: bash topic-condition.sh orders Synced True Synced + - script: test -n "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.topicId}')" + - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.partitions[*].index}')" = "0 1 2" + - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 3"' + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=3600000" + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "max.message.bytes=2097152" diff --git a/tests/templates/kuttl/topic/20-create-topic.yaml b/tests/templates/kuttl/topic/20-create-topic.yaml new file mode 100644 index 00000000..e91eecfb --- /dev/null +++ b/tests/templates/kuttl/topic/20-create-topic.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterName: test-kafka + topicName: orders + partitions: 3 + replicationFactor: 1 + config: + retention.ms: "3600000" + max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/30-assert.yaml b/tests/templates/kuttl/topic/30-assert.yaml new file mode 100644 index 00000000..f8eaa137 --- /dev/null +++ b/tests/templates/kuttl/topic/30-assert.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Synced True Synced + - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.partitions[*].index}')" = "0 1 2 3 4" + - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 5"' + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" diff --git a/tests/templates/kuttl/topic/30-change-topic.yaml b/tests/templates/kuttl/topic/30-change-topic.yaml new file mode 100644 index 00000000..9b4b2667 --- /dev/null +++ b/tests/templates/kuttl/topic/30-change-topic.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterName: test-kafka + topicName: orders + partitions: 5 + replicationFactor: 1 + config: + retention.ms: "7200000" + max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/40-set-config-out-of-band.yaml b/tests/templates/kuttl/topic/40-set-config-out-of-band.yaml new file mode 100644 index 00000000..2a125a28 --- /dev/null +++ b/tests/templates/kuttl/topic/40-set-config-out-of-band.yaml @@ -0,0 +1,6 @@ +--- +# A config set by someone else than the agent, e.g. an administrator with kafka-configs.sh +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: bash kafka-cli.sh kafka-configs.sh --alter --entity-type topics --entity-name orders --add-config segment.ms=86400000 diff --git a/tests/templates/kuttl/topic/41-assert.yaml b/tests/templates/kuttl/topic/41-assert.yaml new file mode 100644 index 00000000..796e09f8 --- /dev/null +++ b/tests/templates/kuttl/topic/41-assert.yaml @@ -0,0 +1,12 @@ +--- +# The key removed from the spec is reset, the key set out of band stays. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Synced True Synced + - script: | + configs=$(bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders) + ! echo "$configs" | grep max.message.bytes + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "segment.ms=86400000" + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" diff --git a/tests/templates/kuttl/topic/41-remove-config-key.yaml b/tests/templates/kuttl/topic/41-remove-config-key.yaml new file mode 100644 index 00000000..88012997 --- /dev/null +++ b/tests/templates/kuttl/topic/41-remove-config-key.yaml @@ -0,0 +1,8 @@ +--- +# kuttl merges applied objects into the existing ones, so a removed key has to be removed explicitly. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + kubectl patch kafkatopic orders --namespace $NAMESPACE --type json + --patch '[{"op": "remove", "path": "/spec/config/max.message.bytes"}]' diff --git a/tests/templates/kuttl/topic/50-assert.yaml b/tests/templates/kuttl/topic/50-assert.yaml new file mode 100644 index 00000000..443a7e5a --- /dev/null +++ b/tests/templates/kuttl/topic/50-assert.yaml @@ -0,0 +1,9 @@ +--- +# The topic stays usable and unchanged. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Synced False UnsupportedChange + - script: bash topic-condition.sh orders Ready True Available + - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 5"' diff --git a/tests/templates/kuttl/topic/50-unsupported-change.yaml b/tests/templates/kuttl/topic/50-unsupported-change.yaml new file mode 100644 index 00000000..72f427bc --- /dev/null +++ b/tests/templates/kuttl/topic/50-unsupported-change.yaml @@ -0,0 +1,13 @@ +--- +# Partitions cannot be removed, and the agent does not reassign replicas. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterName: test-kafka + topicName: orders + partitions: 2 + replicationFactor: 3 + config: + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/51-assert.yaml b/tests/templates/kuttl/topic/51-assert.yaml new file mode 100644 index 00000000..3f2cfc72 --- /dev/null +++ b/tests/templates/kuttl/topic/51-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Synced True Synced diff --git a/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml b/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml new file mode 100644 index 00000000..4a1fa199 --- /dev/null +++ b/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterName: test-kafka + topicName: orders + partitions: 5 + replicationFactor: 1 + config: + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/60-assert.yaml b/tests/templates/kuttl/topic/60-assert.yaml new file mode 100644 index 00000000..d212ade8 --- /dev/null +++ b/tests/templates/kuttl/topic/60-assert.yaml @@ -0,0 +1,11 @@ +--- +# Ready stays unknown, because the agent did not look at the topic in Kafka. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh throttled Synced False InvalidSpec + - script: bash topic-condition.sh throttled Ready "" "" + - script: | + topics=$(bash kafka-cli.sh kafka-topics.sh --list) + ! echo "$topics" | grep -x throttled diff --git a/tests/templates/kuttl/topic/60-invalid-spec.yaml b/tests/templates/kuttl/topic/60-invalid-spec.yaml new file mode 100644 index 00000000..effcf8bc --- /dev/null +++ b/tests/templates/kuttl/topic/60-invalid-spec.yaml @@ -0,0 +1,13 @@ +--- +# Throttles are set by partition reassignments and cannot be managed by a KafkaTopic. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: throttled +spec: + clusterName: test-kafka + topicName: throttled + partitions: 1 + replicationFactor: 1 + config: + leader.replication.throttled.replicas: "*" diff --git a/tests/templates/kuttl/topic/70-create-foreign-topic.yaml b/tests/templates/kuttl/topic/70-create-foreign-topic.yaml new file mode 100644 index 00000000..da72f46a --- /dev/null +++ b/tests/templates/kuttl/topic/70-create-foreign-topic.yaml @@ -0,0 +1,6 @@ +--- +# A topic created outside of KafkaTopics, e.g. by an application or an administrator +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: bash kafka-cli.sh kafka-topics.sh --create --topic legacy --partitions 1 --replication-factor 1 diff --git a/tests/templates/kuttl/topic/71-assert.yaml b/tests/templates/kuttl/topic/71-assert.yaml new file mode 100644 index 00000000..3ed15c31 --- /dev/null +++ b/tests/templates/kuttl/topic/71-assert.yaml @@ -0,0 +1,8 @@ +--- +# The adoption policy defaults to Refuse. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh legacy Ready False AlreadyExists + - script: test -z "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.status.topicId}')" diff --git a/tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml b/tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml new file mode 100644 index 00000000..fe7eef33 --- /dev/null +++ b/tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: legacy +spec: + clusterName: test-kafka + topicName: legacy + partitions: 1 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml b/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml new file mode 100644 index 00000000..ef3a851d --- /dev/null +++ b/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: legacy +spec: + clusterName: test-kafka + topicName: legacy + partitions: 1 + replicationFactor: 1 + managementPolicy: + adoption: Adopt diff --git a/tests/templates/kuttl/topic/72-assert.yaml b/tests/templates/kuttl/topic/72-assert.yaml new file mode 100644 index 00000000..02e00c9f --- /dev/null +++ b/tests/templates/kuttl/topic/72-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh legacy Ready True Available + - script: bash topic-condition.sh legacy Synced True Synced + - script: test -n "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.metadata.annotations.stackable\.tech/owned-since}')" diff --git a/tests/templates/kuttl/topic/80-assert.yaml b/tests/templates/kuttl/topic/80-assert.yaml new file mode 100644 index 00000000..ab8423f0 --- /dev/null +++ b/tests/templates/kuttl/topic/80-assert.yaml @@ -0,0 +1,9 @@ +--- +# The older KafkaTopic keeps managing the topic. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders-copy Ready False Conflict + - script: bash topic-condition.sh orders Ready True Available + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" diff --git a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml new file mode 100644 index 00000000..3a2b28c5 --- /dev/null +++ b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml @@ -0,0 +1,11 @@ +--- +# A second KafkaTopic for the topic already managed by the KafkaTopic orders +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders-copy +spec: + clusterName: test-kafka + topicName: orders + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/81-assert.yaml b/tests/templates/kuttl/topic/81-assert.yaml new file mode 100644 index 00000000..091073c7 --- /dev/null +++ b/tests/templates/kuttl/topic/81-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash kafka-cli.sh kafka-topics.sh --list | grep -x orders diff --git a/tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml b/tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml new file mode 100644 index 00000000..a9805a16 --- /dev/null +++ b/tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders-copy diff --git a/tests/templates/kuttl/topic/90-assert.yaml b/tests/templates/kuttl/topic/90-assert.yaml new file mode 100644 index 00000000..b2bebc87 --- /dev/null +++ b/tests/templates/kuttl/topic/90-assert.yaml @@ -0,0 +1,8 @@ +--- +# Only KafkaTopics that delete their topic in Kafka have a finalizer. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.metadata.finalizers}')" = '["kafka.stackable.tech/kafkatopic"]' + - script: test -z "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.metadata.finalizers}')" diff --git a/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml b/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml new file mode 100644 index 00000000..2ea201fe --- /dev/null +++ b/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: legacy +spec: + clusterName: test-kafka + topicName: legacy + partitions: 1 + replicationFactor: 1 + managementPolicy: + adoption: Adopt + deletion: Delete diff --git a/tests/templates/kuttl/topic/91-assert.yaml b/tests/templates/kuttl/topic/91-assert.yaml new file mode 100644 index 00000000..537525c9 --- /dev/null +++ b/tests/templates/kuttl/topic/91-assert.yaml @@ -0,0 +1,11 @@ +--- +# orders has the default deletion policy Retain, legacy has Delete. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: test -z "$(kubectl get kafkatopics --namespace $NAMESPACE --output name)" + - script: bash kafka-cli.sh kafka-topics.sh --list | grep -x orders + - script: | + topics=$(bash kafka-cli.sh kafka-topics.sh --list) + ! echo "$topics" | grep -x legacy diff --git a/tests/templates/kuttl/topic/91-delete-topics.yaml b/tests/templates/kuttl/topic/91-delete-topics.yaml new file mode 100644 index 00000000..430f5841 --- /dev/null +++ b/tests/templates/kuttl/topic/91-delete-topics.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: legacy + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: throttled diff --git a/tests/templates/kuttl/topic/95-disable-platform-access.yaml.j2 b/tests/templates/kuttl/topic/95-disable-platform-access.yaml.j2 new file mode 100644 index 00000000..5e9b635e --- /dev/null +++ b/tests/templates/kuttl/topic/95-disable-platform-access.yaml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + image: +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls + platformAccess: + enabled: false + authentication: + anonymous: {} + controllers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 + brokers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/topic/95-errors.yaml b/tests/templates/kuttl/topic/95-errors.yaml new file mode 100644 index 00000000..1769b9a5 --- /dev/null +++ b/tests/templates/kuttl/topic/95-errors.yaml @@ -0,0 +1,16 @@ +--- +# The agent resources are removed when platform access is disabled. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-kafka-agent +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-kafka-agent +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-kafka-agent diff --git a/tests/templates/kuttl/topic/kafka-cli.sh b/tests/templates/kuttl/topic/kafka-cli.sh new file mode 100644 index 00000000..31627437 --- /dev/null +++ b/tests/templates/kuttl/topic/kafka-cli.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Usage: kafka-cli.sh , e.g. kafka-cli.sh kafka-topics.sh --describe --topic orders +# Runs a Kafka CLI tool in the broker Pod against the brokers. +set -euo pipefail + +# The script is expanded in the broker Pod, not here. +# shellcheck disable=SC2016 +kubectl exec --namespace "$NAMESPACE" test-kafka-broker-default-0 --container kafka -- bash -c ' + bootstrap="$(cat /stackable/listener-broker/default-address/address):$(cat /stackable/listener-broker/default-address/ports/kafka-tls)" + tool=$1 + shift + "/stackable/kafka/bin/$tool" --bootstrap-server "$bootstrap" --command-config /stackable/config/client.properties "$@" +' kafka-cli "$@" diff --git a/tests/templates/kuttl/topic/topic-condition.sh b/tests/templates/kuttl/topic/topic-condition.sh new file mode 100644 index 00000000..59a05a0d --- /dev/null +++ b/tests/templates/kuttl/topic/topic-condition.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Usage: topic-condition.sh +# Fails unless the condition of the KafkaTopic has the given status and reason. +set -euo pipefail + +actual=$(kubectl get kafkatopic --namespace "$NAMESPACE" "$1" \ + --output "jsonpath={.status.conditions[?(@.type==\"$2\")].status}/{.status.conditions[?(@.type==\"$2\")].reason}") +if [ "$actual" != "$3/$4" ]; then + echo "$1: expected the condition $2 to be $3/$4, but it is \"$actual\"" + exit 1 +fi diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index c4c294ff..6d835258 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -7,7 +7,7 @@ dimensions: - name: kafka-kraft values: - 3.9.2 - - 4.3.1 + # - 4.3.1 - name: kafka values: - 3.9.2 @@ -72,6 +72,12 @@ dimensions: - "cluster-internal" - "external-stable" - "external-unstable" + # How the agent authenticates to Kafka + - name: platform-access-authentication + values: + - anonymous + - secret-class + - secret tests: - name: operations-kraft dimensions: @@ -81,6 +87,15 @@ tests: dimensions: - kafka-kraft - openshift + - name: topic + dimensions: + - kafka-kraft + - openshift + - name: topic-authentication + dimensions: + - kafka-kraft + - platform-access-authentication + - openshift - name: smoke dimensions: - kafka From f094a2317a3c4bd2c23fef15f40181dbd6113a5e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 26 Oct 2026 14:56:33 +0100 Subject: [PATCH 09/33] update the CRD to reflect operator-rs changes and the CRD decision --- extra/crds.yaml | 150 ++++++++++++++----------- rust/shared/src/crd/topic.rs | 210 ++++++++++++++++++++++++++--------- 2 files changed, 241 insertions(+), 119 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index 1824764a..96d539fe 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2269,76 +2269,78 @@ spec: description: |- A Kafka topic managed by the agent of the referenced Kafka cluster. - The referenced `KafkaCluster` must have `platformAccess` enabled or this resource will - not be reconciled. + The referenced `KafkaCluster` must have `clusterConfig.managementAccess` enabled or this + resource will not be reconciled. properties: - clusterName: - description: Name of a KafkaCluster in the namespace of this KafkaTopic resource. Immutable. - maxLength: 40 - minLength: 1 - type: string + clusterRef: + description: The KafkaCluster managing this topic. Immutable. + properties: + name: + description: Name of a KafkaCluster in the namespace of this KafkaTopic. + maxLength: 40 + minLength: 1 + type: string + required: + - name + type: object x-kubernetes-validations: - - message: clusterName is immutable + - message: clusterRef is immutable rule: self == oldSelf - config: + configOverrides: additionalProperties: type: string default: {} description: |- - Topic configs like `retention.ms`. Keys removed from this map are reset to the broker - default. Keys not listed here are not managed by the agent. - type: object - managementPolicy: - default: - adoption: Refuse - deletion: Retain - description: What an agent may do with a resource inside the product beyond applying its spec. - properties: - adoption: - default: Refuse - description: |- - Whether a resource that already exists in the product, but was not created by the - agent, is taken over. `Refuse` by default. - enum: - - Refuse - - Adopt - type: string - deletion: - default: Retain - description: |- - What happens to the resource in the product when this object is deleted. `Retain` by - default. - enum: - - Retain - - Delete - type: string + Topic configs like `retention.ms`. All other topic-level configs are reset to the broker + defaults, except the replication throttles set by partition reassignments. type: object + deletionPolicy: + default: Retain + description: What happens to a resource in the product when the object managing it is deleted. + enum: + - Retain + - Delete + type: string partitions: - description: Number of partitions. Can only be increased. + description: |- + Number of partitions. Can only be increased. Defaults to the broker setting + `num.partitions` when the topic is created. format: int32 minimum: 1.0 + nullable: true type: integer replicationFactor: - description: Number of replicas of each partition. Changes are not applied to existing topics. + description: |- + Number of replicas of each partition. Changes are not applied to existing topics. + Defaults to the broker setting `default.replication.factor` when the topic is created. format: int16 maximum: 32767.0 minimum: 1.0 + nullable: true type: integer topicName: - description: Name of the topic in Kafka. Immutable. + description: Name of the topic in Kafka. Defaults to `metadata.name`. Immutable. maxLength: 249 minLength: 1 - pattern: ^([a-zA-Z0-9-][a-zA-Z0-9._-]*|_([a-zA-Z0-9.-][a-zA-Z0-9._-]*)?|\.[a-zA-Z0-9_-][a-zA-Z0-9._-]*|\.\.[a-zA-Z0-9._-]+)$ + nullable: true + pattern: ^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$ type: string - x-kubernetes-validations: - - message: topicName is immutable - rule: self == oldSelf + topicOperation: + default: + reconciliationPaused: false + description: Operations on this topic, independent of the `clusterOperation` of the KafkaCluster. + properties: + reconciliationPaused: + default: false + description: Stops the agent from changing the topic in Kafka. The status is still updated. + type: boolean + type: object required: - - clusterName - - partitions - - replicationFactor - - topicName + - clusterRef type: object + x-kubernetes-validations: + - message: topicName is immutable + rule: self.?topicName == oldSelf.?topicName status: nullable: true properties: @@ -2376,36 +2378,54 @@ spec: - type type: object type: array - partitions: - default: [] - description: The partitions of the topic in Kafka. - items: + config: + additionalProperties: properties: - index: - format: int32 - type: integer - replicas: - description: |- - Broker ids holding a replica, the preferred leader first. During a reassignment, this - includes the replicas being added and removed. - items: - format: int32 - type: integer - type: array + source: + description: Where the value comes from, like `DYNAMIC_TOPIC_CONFIG` or `DEFAULT_CONFIG`. + type: string + value: + description: The value, not set for sensitive configs. + nullable: true + type: string required: - - index - - replicas + - source type: object - type: array + default: {} + description: All effective topic configs in Kafka. + type: object + partitions: + description: The number of partitions in Kafka. + format: int32 + nullable: true + type: integer + replicationFactor: + description: |- + The replication factor, not counting replicas that a running reassignment removes. + Not set if the partitions have different replication factors. + format: int16 + maximum: 32767.0 + minimum: -32768.0 + nullable: true + type: integer topicId: description: The topic id in Kafka, as shown by `kafka-topics.sh`. nullable: true type: string + topicName: + description: The name of the topic in Kafka. + maxLength: 249 + minLength: 1 + nullable: true + pattern: ^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$ + type: string type: object required: - spec title: KafkaTopic type: object + selectableFields: + - jsonPath: .spec.clusterRef.name served: true storage: true subresources: diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 3a9ef668..19bd870d 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -1,23 +1,23 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr}; use serde::{Deserialize, Serialize}; use stackable_operator::{ attributed_string_type, - crd::platform_access::v1alpha1::ManagementPolicy, + crd::management_access::v1alpha1::DeletionPolicy, k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition, - kube::CustomResource, + kube::{CustomResource, ResourceExt}, schemars::{self, JsonSchema}, - v2::types::operator::ClusterName, + v2::{macros::attributed_string_type::Error as TopicNameError, types::operator::ClusterName}, versioned::versioned, }; attributed_string_type! { TopicName, - "The name of a Kafka topic, not `.` or `..` and not starting with `__`", + "The name of a Kafka topic, not `.` or `..`", "shop.orders", (min_length = 1), (max_length = 249), - (regex = r"^([a-zA-Z0-9-][a-zA-Z0-9._-]*|_([a-zA-Z0-9.-][a-zA-Z0-9._-]*)?|\.[a-zA-Z0-9_-][a-zA-Z0-9._-]*|\.\.[a-zA-Z0-9._-]+)$") + (regex = r"^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$") } #[versioned( @@ -34,47 +34,73 @@ pub mod versioned { /// A Kafka topic managed by the agent of the referenced Kafka cluster. /// /// The referenced `KafkaCluster` must have `clusterConfig.managementAccess` enabled or this - /// resource will - /// not be reconciled. + /// resource will not be reconciled. #[versioned(crd( doc = "A Kafka topic, created and updated by the agent of the referenced Kafka cluster.", group = "kafka.stackable.tech", plural = "kafkatopics", status = "KafkaTopicStatus", + selectable = ".spec.clusterRef.name", namespaced ))] #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "self.?topicName == oldSelf.?topicName", + "message": "topicName is immutable" + }]))] pub struct KafkaTopicSpec { - /// Name of a KafkaCluster in the namespace of this KafkaTopic resource. Immutable. + /// The KafkaCluster managing this topic. Immutable. #[schemars(extend("x-kubernetes-validations" = [{ "rule": "self == oldSelf", - "message": "clusterName is immutable" + "message": "clusterRef is immutable" }]))] - pub cluster_name: ClusterName, + pub cluster_ref: v1alpha1::KafkaClusterRef, - #[serde(default)] - pub management_policy: ManagementPolicy, - - /// Name of the topic in Kafka. Immutable. - #[schemars(extend("x-kubernetes-validations" = [{ - "rule": "self == oldSelf", - "message": "topicName is immutable" - }]))] - pub topic_name: TopicName, + /// Name of the topic in Kafka. Defaults to `metadata.name`. Immutable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_name: Option, - /// Number of partitions. Can only be increased. + /// Number of partitions. Can only be increased. Defaults to the broker setting + /// `num.partitions` when the topic is created. + #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 1))] - pub partitions: i32, + pub partitions: Option, /// Number of replicas of each partition. Changes are not applied to existing topics. + /// Defaults to the broker setting `default.replication.factor` when the topic is created. + #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 1))] - pub replication_factor: i16, + pub replication_factor: Option, + + /// Topic configs like `retention.ms`. All other topic-level configs are reset to the broker + /// defaults, except the replication throttles set by partition reassignments. + #[serde(default)] + pub config_overrides: BTreeMap, - /// Topic configs like `retention.ms`. Keys removed from this map are reset to the broker - /// default. Keys not listed here are not managed by the agent. + // no doc - docs in DeletionPolicy enum. #[serde(default)] - pub config: BTreeMap, + pub deletion_policy: DeletionPolicy, + + // no doc - docs in TopicOperation struct. + #[serde(default)] + pub topic_operation: v1alpha1::TopicOperation, + } + + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaClusterRef { + /// Name of a KafkaCluster in the namespace of this KafkaTopic. + pub name: ClusterName, + } + + /// Operations on this topic, independent of the `clusterOperation` of the KafkaCluster. + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct TopicOperation { + /// Stops the agent from changing the topic in Kafka. The status is still updated. + #[serde(default)] + pub reconciliation_paused: bool, } #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -88,58 +114,105 @@ pub mod versioned { #[serde(default, skip_serializing_if = "Option::is_none")] pub topic_id: Option, - /// The partitions of the topic in Kafka. + /// The name of the topic in Kafka. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_name: Option, + + /// The number of partitions in Kafka. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub partitions: Option, + + /// The replication factor, not counting replicas that a running reassignment removes. + /// Not set if the partitions have different replication factors. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replication_factor: Option, + + /// All effective topic configs in Kafka. #[serde(default)] - pub partitions: Vec, + pub config: BTreeMap, } #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] - pub struct KafkaTopicPartitionStatus { - pub index: i32, + pub struct KafkaTopicConfigStatus { + /// The value, not set for sensitive configs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, - /// Broker ids holding a replica, the preferred leader first. During a reassignment, this - /// includes the replicas being added and removed. - pub replicas: Vec, + /// Where the value comes from, like `DYNAMIC_TOPIC_CONFIG` or `DEFAULT_CONFIG`. + pub source: String, + } +} + +impl v1alpha1::KafkaTopic { + /// The name of the topic in Kafka: `spec.topicName`, otherwise `metadata.name`. + /// + /// Fails if `metadata.name` is longer than a topic name may be. + pub fn topic_name(&self) -> Result { + match &self.spec.topic_name { + Some(topic_name) => Ok(topic_name.clone()), + None => TopicName::from_str(&self.name_any()), + } } } #[cfg(test)] mod tests { - use std::str::FromStr; - use rstest::rstest; - use stackable_operator::versioned::test_utils::RoundtripTestData; + use stackable_operator::{kube::api::ObjectMeta, versioned::test_utils::RoundtripTestData}; use super::*; impl RoundtripTestData for v1alpha1::KafkaTopicSpec { fn roundtrip_test_data() -> Vec { stackable_operator::utils::yaml_from_str_singleton_map(indoc::indoc! {r#" - - clusterName: simple-kafka - managementPolicy: - adoption: Adopt - deletion: Delete + - clusterRef: + name: simple-kafka topicName: shop.orders partitions: 6 replicationFactor: 3 - config: + configOverrides: retention.ms: "604800000" - - clusterName: simple-kafka - topicName: minimal - partitions: 1 - replicationFactor: 1 + deletionPolicy: Delete + topicOperation: + reconciliationPaused: true + - clusterRef: + name: simple-kafka "#}) .expect("the roundtrip test data should be valid YAML") } } + fn topic(metadata_name: &str, topic_name: Option<&str>) -> v1alpha1::KafkaTopic { + v1alpha1::KafkaTopic { + metadata: ObjectMeta { + name: Some(metadata_name.to_owned()), + ..ObjectMeta::default() + }, + spec: v1alpha1::KafkaTopicSpec { + cluster_ref: v1alpha1::KafkaClusterRef { + name: ClusterName::from_str("simple-kafka") + .expect("the cluster name should be valid"), + }, + topic_name: topic_name.map(TopicName::from_str_unsafe), + partitions: None, + replication_factor: None, + config_overrides: BTreeMap::new(), + deletion_policy: DeletionPolicy::default(), + topic_operation: v1alpha1::TopicOperation::default(), + }, + status: None, + } + } + #[rstest] #[case::dotted("shop.orders")] #[case::uppercase_and_underscore("Shop_Orders-1")] #[case::single_underscore("_internal")] #[case::underscore_only("_")] + #[case::internal_topic("__consumer_offsets")] #[case::leading_dot(".hidden")] + #[case::trailing_dot("a.")] #[case::three_dots("...")] fn accept_topic_name(#[case] name: &str) { assert!( @@ -152,7 +225,6 @@ mod tests { #[case::empty("")] #[case::dot(".")] #[case::two_dots("..")] - #[case::double_underscore_prefix("__consumer_offsets")] #[case::invalid_character("shop/orders")] #[case::too_long(&"a".repeat(250))] fn reject_topic_name(#[case] name: &str) { @@ -162,6 +234,27 @@ mod tests { ); } + #[rstest] + #[case::explicit("my-topic", Some("shop.orders"), "shop.orders")] + #[case::from_metadata_name("shop.orders", None, "shop.orders")] + fn effective_topic_name( + #[case] metadata_name: &str, + #[case] topic_name: Option<&str>, + #[case] expected: &str, + ) { + assert_eq!( + TopicName::from_str_unsafe(expected), + topic(metadata_name, topic_name) + .topic_name() + .expect("the topic name should be valid") + ); + } + + #[test] + fn reject_too_long_metadata_name_as_topic_name() { + assert!(topic(&"a".repeat(250), None).topic_name().is_err()); + } + #[test] fn schema_contains_validations() { let crd = serde_json::to_value( @@ -169,19 +262,28 @@ mod tests { .expect("the CRD should be mergeable"), ) .expect("the CRD should be serializable"); - let spec = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]["properties"]; + let version = &crd["spec"]["versions"][0]; + let spec = &version["schema"]["openAPIV3Schema"]["properties"]["spec"]; + let properties = &spec["properties"]; assert_eq!( - "self == oldSelf", - spec["clusterName"]["x-kubernetes-validations"][0]["rule"] + ".spec.clusterRef.name", + version["selectableFields"][0]["jsonPath"] ); assert_eq!( "self == oldSelf", - spec["topicName"]["x-kubernetes-validations"][0]["rule"] + properties["clusterRef"]["x-kubernetes-validations"][0]["rule"] + ); + assert_eq!( + "self.?topicName == oldSelf.?topicName", + spec["x-kubernetes-validations"][0]["rule"] + ); + assert_eq!(249, properties["topicName"]["maxLength"]); + assert!(properties["topicName"]["pattern"].is_string()); + assert_eq!(Some(1.0), properties["partitions"]["minimum"].as_f64()); + assert_eq!( + Some(1.0), + properties["replicationFactor"]["minimum"].as_f64() ); - assert_eq!(249, spec["topicName"]["maxLength"]); - assert!(spec["topicName"]["pattern"].is_string()); - assert_eq!(Some(1.0), spec["partitions"]["minimum"].as_f64()); - assert_eq!(Some(1.0), spec["replicationFactor"]["minimum"].as_f64()); } } From d028aa2af4914c0ca20edac5593788bb6780337a Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 26 Oct 2026 21:17:26 +0100 Subject: [PATCH 10/33] rewrite the topic agent for the new KafkaTopic CRD --- rust/agent-binary/src/main.rs | 4 +- rust/agent-binary/src/topic/condition.rs | 349 ++++++---- rust/agent-binary/src/topic/controller.rs | 640 ++---------------- rust/agent-binary/src/topic/finalizer.rs | 137 ++++ .../src/topic/managed_config_keys.rs | 69 -- rust/agent-binary/src/topic/mod.rs | 5 +- rust/agent-binary/src/topic/plan.rs | 463 ++++++++----- rust/agent-binary/src/topic/sync.rs | 280 ++++++++ rust/agent-binary/src/topic/test_support.rs | 23 +- rust/agent-binary/src/topic/update_status.rs | 196 ++++++ rust/agent-binary/src/topic/validate.rs | 234 +++++++ 11 files changed, 1450 insertions(+), 950 deletions(-) create mode 100644 rust/agent-binary/src/topic/finalizer.rs delete mode 100644 rust/agent-binary/src/topic/managed_config_keys.rs create mode 100644 rust/agent-binary/src/topic/sync.rs create mode 100644 rust/agent-binary/src/topic/update_status.rs create mode 100644 rust/agent-binary/src/topic/validate.rs diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index b0bbe1ae..5b61c878 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -115,7 +115,7 @@ async fn main() -> anyhow::Result<()> { client.get_api::>(cluster_namespace.as_ref()), watcher::Config::default(), ); - let topic_store = topic_controller.store(); + let kafka_topic_store = topic_controller.store(); topic_controller .graceful_shutdown_on(sigterm_watcher.handle()) .run( @@ -125,7 +125,7 @@ async fn main() -> anyhow::Result<()> { client, cluster_name, security, - topic_store, + kafka_topic_store, )), ) .for_each_concurrent( diff --git a/rust/agent-binary/src/topic/condition.rs b/rust/agent-binary/src/topic/condition.rs index d29c651d..b8d1eb73 100644 --- a/rust/agent-binary/src/topic/condition.rs +++ b/rust/agent-binary/src/topic/condition.rs @@ -1,22 +1,28 @@ //! The `Ready` and `Synced` conditions of a `KafkaTopic` for the outcome of a sync. use stackable_kafka_admin::client; -use stackable_operator::commons::platform_access::condition::{ - ManagedCondition, ManagedConditionType, +use stackable_operator::status::condition::{ + ClusterConditionStatus, + managed_resource::{ManagedCondition, ManagedConditionType}, }; -const MAX_LISTED_PARTITIONS: usize = 10; +use super::plan::{TopicState, UnsupportedChange}; + +/// The operator writes the bootstrap servers to the discovery ConfigMap only once the +/// listener-operator has assigned addresses to the bootstrap Listeners. +const CLUSTER_NOT_READY_MESSAGE: &str = "the KafkaCluster has no bootstrap servers yet"; #[derive(Debug, Eq, PartialEq)] pub enum TopicSyncOutcome { /// The topic matches the spec Synced { - partitions: usize, - reassigning_partitions: Vec, + reassigning_partition_count: usize, }, + /// The topic matches the spec, apart from the changes the agent does not apply UnsupportedChange { - messages: Vec, + changes: Vec, }, + ReconciliationPaused, ClusterNotReady, InvalidSpec { message: String, @@ -25,11 +31,6 @@ pub enum TopicSyncOutcome { Conflict { message: String, }, - /// The topic exists in Kafka, but this `KafkaTopic` does not own it and is not allowed to adopt - /// it - AlreadyExists { - message: String, - }, KafkaRejected { message: String, }, @@ -49,55 +50,116 @@ impl TopicSyncOutcome { } } -/// The conditions to update. `Ready` says whether the topic is usable through this `KafkaTopic` -/// and is only updated if that is known. -pub fn conditions(outcome: &TopicSyncOutcome) -> Vec { - let available = ready(true, "Available", "the topic exists in Kafka"); - match outcome { - TopicSyncOutcome::Synced { - partitions, - reassigning_partitions, - } => { - let synced = if reassigning_partitions.is_empty() { - synced(true, "Synced", "the topic matches the spec") - } else { - synced( - true, - "ReassignmentInProgress", - &reassignment_message(reassigning_partitions, *partitions), - ) - }; - vec![available, synced] +/// `topic_state` is the topic in Kafka, if it exists and is managed by this `KafkaTopic`. +pub fn conditions_from( + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> Vec { + vec![ + ready_from(sync_outcome, topic_state), + synced_from(sync_outcome, topic_state), + ] +} + +/// Whether the topic exists in Kafka, `Unknown` if the agent could not find out. +fn ready_from( + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> ManagedCondition { + use ClusterConditionStatus::{False, True, Unknown}; + + // Also after a failed change, because the topic was observed before. + if let Some(topic_state) = topic_state { + return ready( + True, + "Available", + &format!( + "Topic {} exists with {} partitions", + topic_state.name, topic_state.partition_count + ), + ); + } + match sync_outcome { + TopicSyncOutcome::Synced { .. } + | TopicSyncOutcome::UnsupportedChange { .. } + | TopicSyncOutcome::ReconciliationPaused => { + ready(False, "Unavailable", "the topic does not exist in Kafka") + } + TopicSyncOutcome::Conflict { .. } => { + ready(False, "Conflict", "another KafkaTopic manages the topic") } - TopicSyncOutcome::UnsupportedChange { messages } => vec![ - available, - synced(false, "UnsupportedChange", &messages.join("; ")), - ], TopicSyncOutcome::ClusterNotReady => { - let message = "the KafkaCluster has no bootstrap servers yet"; - vec![ - ready(false, "ClusterNotReady", message), - synced(false, "ClusterNotReady", message), - ] + ready(Unknown, "ClusterNotReady", CLUSTER_NOT_READY_MESSAGE) + } + TopicSyncOutcome::InvalidSpec { message } => ready(Unknown, "InvalidSpec", message), + TopicSyncOutcome::KafkaRejected { message } => ready(Unknown, "KafkaRejected", message), + TopicSyncOutcome::ConnectionFailed { message } => { + ready(Unknown, "ConnectionFailed", message) } - TopicSyncOutcome::Conflict { message } => vec![ - ready(false, "Conflict", "another KafkaTopic manages the topic"), - synced(false, "Conflict", message), - ], - TopicSyncOutcome::AlreadyExists { message } => vec![ - ready( - false, - "AlreadyExists", - "the topic exists in Kafka, but is not owned by this KafkaTopic", + } +} + +/// Whether the last sync applied the spec to the topic. +fn synced_from( + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> ManagedCondition { + use ClusterConditionStatus::{False, True}; + + match sync_outcome { + TopicSyncOutcome::Synced { + reassigning_partition_count, + } => match (reassigning_partition_count, topic_state) { + (0, _) | (_, None) => synced(True, "Synced", "the topic matches the spec"), + (reassigning, Some(topic_state)) => synced( + True, + "ReassignmentInProgress", + &format!( + "{reassigning} of {} partitions are being reassigned", + topic_state.partition_count + ), ), - synced(false, "AlreadyExists", message), - ], - TopicSyncOutcome::InvalidSpec { message } => vec![synced(false, "InvalidSpec", message)], - TopicSyncOutcome::KafkaRejected { message } => { - vec![synced(false, "KafkaRejected", message)] + }, + TopicSyncOutcome::UnsupportedChange { changes } => { + let messages = changes.iter().map(message_from).collect::>(); + synced(False, "UnsupportedChange", &messages.join("; ")) + } + TopicSyncOutcome::ReconciliationPaused => synced( + False, + "ReconciliationPaused", + "the reconciliation is paused, only the status is updated", + ), + TopicSyncOutcome::Conflict { message } => synced(False, "Conflict", message), + TopicSyncOutcome::ClusterNotReady => { + synced(False, "ClusterNotReady", CLUSTER_NOT_READY_MESSAGE) } + TopicSyncOutcome::InvalidSpec { message } => synced(False, "InvalidSpec", message), + TopicSyncOutcome::KafkaRejected { message } => synced(False, "KafkaRejected", message), TopicSyncOutcome::ConnectionFailed { message } => { - vec![synced(false, "ConnectionFailed", message)] + synced(False, "ConnectionFailed", message) + } + } +} + +fn message_from(unsupported_change: &UnsupportedChange) -> String { + match unsupported_change { + UnsupportedChange::DecreasePartitions { desired, actual } => { + format!("partitions cannot be decreased (spec: {desired}, actual: {actual})") + } + UnsupportedChange::ChangeReplicationFactor { + desired, + actual_min, + actual_max, + } => { + let actual = if actual_min == actual_max { + actual_min.to_string() + } else { + format!("between {actual_min} and {actual_max}") + }; + format!( + "the replication factor cannot be changed (spec: {desired}, actual: {actual}); \ + change it with kafka-reassign-partitions" + ) } } } @@ -112,34 +174,17 @@ fn error_chain(error: &dyn std::error::Error) -> String { .join(": ") } -fn reassignment_message(reassigning_partitions: &[i32], partitions: usize) -> String { - let listed = reassigning_partitions - .iter() - .take(MAX_LISTED_PARTITIONS) - .map(i32::to_string) - .collect::>() - .join(", "); - let more = reassigning_partitions - .len() - .saturating_sub(MAX_LISTED_PARTITIONS); - if more > 0 { - format!("partitions {listed} and {more} more of {partitions} are being reassigned") - } else { - format!("partitions {listed} of {partitions} are being reassigned") - } -} - -fn ready(status: bool, reason: &str, message: &str) -> ManagedCondition { +fn ready(status: ClusterConditionStatus, reason: &str, message: &str) -> ManagedCondition { condition(ManagedConditionType::Ready, status, reason, message) } -fn synced(status: bool, reason: &str, message: &str) -> ManagedCondition { +fn synced(status: ClusterConditionStatus, reason: &str, message: &str) -> ManagedCondition { condition(ManagedConditionType::Synced, status, reason, message) } fn condition( type_: ManagedConditionType, - status: bool, + status: ClusterConditionStatus, reason: &str, message: &str, ) -> ManagedCondition { @@ -153,130 +198,180 @@ fn condition( #[cfg(test)] mod tests { + use std::{collections::BTreeMap, str::FromStr}; + use snafu::Snafu; + use stackable_kafka_shared::crd::topic::TopicName; use super::*; + use crate::topic::condition::ClusterConditionStatus::{False, True, Unknown}; - fn summary(conditions: &[ManagedCondition]) -> Vec<(String, bool, &str)> { + fn summary(conditions: &[ManagedCondition]) -> Vec<(String, ClusterConditionStatus, &str)> { conditions .iter() .map(|condition| { ( condition.type_.to_string(), - condition.status, + condition.status.clone(), condition.reason.as_str(), ) }) .collect() } + fn topic_state(partition_count: i32) -> TopicState { + TopicState { + name: TopicName::from_str("orders").expect("the topic name should be valid"), + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partition_count, + replication_factor: Some(1), + effective_configs: BTreeMap::new(), + } + } + #[test] fn mark_a_synced_topic() { - let conditions = conditions(&TopicSyncOutcome::Synced { - partitions: 6, - reassigning_partitions: vec![], - }); + let conditions = conditions_from( + &TopicSyncOutcome::Synced { + reassigning_partition_count: 0, + }, + Some(&topic_state(6)), + ); assert_eq!( vec![ - ("Ready".to_owned(), true, "Available"), - ("Synced".to_owned(), true, "Synced") + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), True, "Synced") ], summary(&conditions) ); + assert_eq!( + "Topic orders exists with 6 partitions", + conditions[0].message + ); } #[test] fn keep_a_reassigned_topic_synced() { - let conditions = conditions(&TopicSyncOutcome::Synced { - partitions: 6, - reassigning_partitions: vec![0, 2], - }); + let conditions = conditions_from( + &TopicSyncOutcome::Synced { + reassigning_partition_count: 3, + }, + Some(&topic_state(6)), + ); assert_eq!( vec![ - ("Ready".to_owned(), true, "Available"), - ("Synced".to_owned(), true, "ReassignmentInProgress") + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), True, "ReassignmentInProgress") ], summary(&conditions) ); assert_eq!( - "partitions 0, 2 of 6 are being reassigned", + "3 of 6 partitions are being reassigned", conditions[1].message ); } #[test] - fn limit_the_listed_reassigning_partitions() { - let message = reassignment_message(&(0..12).collect::>(), 20); + fn mark_an_unsupported_change() { + let conditions = conditions_from( + &TopicSyncOutcome::UnsupportedChange { + changes: vec![ + UnsupportedChange::DecreasePartitions { + desired: 4, + actual: 6, + }, + UnsupportedChange::ChangeReplicationFactor { + desired: 2, + actual_min: 2, + actual_max: 3, + }, + ], + }, + Some(&topic_state(6)), + ); assert_eq!( - "partitions 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 and 2 more of 20 are being reassigned", - message + vec![ + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), False, "UnsupportedChange") + ], + summary(&conditions) + ); + assert_eq!( + "partitions cannot be decreased (spec: 4, actual: 6); the replication factor cannot \ + be changed (spec: 2, actual: between 2 and 3); change it with \ + kafka-reassign-partitions", + conditions[1].message ); } #[test] - fn only_update_synced_after_an_error() { - let conditions = conditions(&TopicSyncOutcome::ConnectionFailed { - message: "timed out".to_owned(), - }); + fn report_a_missing_topic_while_paused() { + let conditions = conditions_from(&TopicSyncOutcome::ReconciliationPaused, None); assert_eq!( - vec![("Synced".to_owned(), false, "ConnectionFailed")], + vec![ + ("Ready".to_owned(), False, "Unavailable"), + ("Synced".to_owned(), False, "ReconciliationPaused") + ], summary(&conditions) ); } #[test] - fn mark_an_unsupported_change() { - let conditions = conditions(&TopicSyncOutcome::UnsupportedChange { - messages: vec!["first".to_owned(), "second".to_owned()], - }); + fn mark_a_conflict_as_not_ready() { + let conditions = conditions_from( + &TopicSyncOutcome::Conflict { + message: "managed by older".to_owned(), + }, + None, + ); assert_eq!( vec![ - ("Ready".to_owned(), true, "Available"), - ("Synced".to_owned(), false, "UnsupportedChange") + ("Ready".to_owned(), False, "Conflict"), + ("Synced".to_owned(), False, "Conflict") ], summary(&conditions) ); - assert_eq!("first; second", conditions[1].message); } - #[test] - fn leave_ready_untouched_for_an_invalid_spec() { - let conditions = conditions(&TopicSyncOutcome::InvalidSpec { - message: "invalid".to_owned(), - }); - + #[rstest::rstest] + #[case::cluster_not_ready(TopicSyncOutcome::ClusterNotReady, "ClusterNotReady")] + #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: "invalid".to_owned() }, "InvalidSpec")] + #[case::kafka_rejected(TopicSyncOutcome::KafkaRejected { message: "denied".to_owned() }, "KafkaRejected")] + #[case::connection_failed(TopicSyncOutcome::ConnectionFailed { message: "timed out".to_owned() }, "ConnectionFailed")] + fn mark_ready_as_unknown_if_the_topic_cannot_be_observed( + #[case] sync_outcome: TopicSyncOutcome, + #[case] reason: &str, + ) { assert_eq!( - vec![("Synced".to_owned(), false, "InvalidSpec")], - summary(&conditions) + vec![ + ("Ready".to_owned(), Unknown, reason), + ("Synced".to_owned(), False, reason) + ], + summary(&conditions_from(&sync_outcome, None)) ); } #[test] - fn keep_the_ready_message_fixed_for_a_foreign_topic() { - let conditions = conditions(&TopicSyncOutcome::AlreadyExists { - message: "set managementPolicy.adoption to Adopt".to_owned(), - }); + fn keep_an_observed_topic_ready_if_a_change_fails() { + let conditions = conditions_from( + &TopicSyncOutcome::KafkaRejected { + message: "denied".to_owned(), + }, + Some(&topic_state(6)), + ); assert_eq!( vec![ - ("Ready".to_owned(), false, "AlreadyExists"), - ("Synced".to_owned(), false, "AlreadyExists") + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), False, "KafkaRejected") ], summary(&conditions) ); - assert_eq!( - "the topic exists in Kafka, but is not owned by this KafkaTopic", - conditions[0].message - ); - assert_eq!( - "set managementPolicy.adoption to Adopt", - conditions[1].message - ); } #[derive(Debug, Snafu)] diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs index 484bd9e1..105ac03c 100644 --- a/rust/agent-binary/src/topic/controller.rs +++ b/rust/agent-binary/src/topic/controller.rs @@ -1,26 +1,13 @@ -//! The `KafkaTopic` controller: reconcile, cleanup on deletion and error policy. +//! The `KafkaTopic` controller: reconcile and error policy. -use std::{collections::BTreeSet, path::Path, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; use const_format::concatcp; -use serde_json::json; use snafu::{ResultExt, Snafu}; -use stackable_kafka_admin::{ - bootstrap::BrokerAddress, - client::{self, KafkaAdminClient, Topic}, - connection::Security, -}; -use stackable_kafka_shared::crd::topic::v1alpha1::{ - KafkaTopic, KafkaTopicPartitionStatus, KafkaTopicStatus, -}; +use stackable_kafka_admin::{client, connection::Security}; +use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; use stackable_operator::{ - commons::platform_access::{ - condition::update_conditions, - ownership::{OWNED_SINCE_ANNOTATION, is_owned}, - }, - k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, kube::{ - ResourceExt, core::{DeserializeGuard, error_boundary}, runtime::{controller::Action, reflector::Store}, }, @@ -32,31 +19,22 @@ use tokio::sync::Mutex; use tracing::info; use super::{ - condition::{TopicSyncOutcome, conditions}, - managed_config_keys::{MANAGED_CONFIG_KEYS_ANNOTATION, managed_config_keys, render}, - plan::{ObservedTopic, TopicPlan, plan}, -}; -use crate::{ - AGENT_NAME, - config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}, + condition::TopicSyncOutcome, + finalizer::{add_finalizer, reconcile_finalizer}, + sync::{CachedClient, sync}, + update_status::update_status, }; +use crate::AGENT_NAME; pub const TOPIC_CONTROLLER_NAME: &str = "kafkatopic"; pub const TOPIC_FULL_CONTROLLER_NAME: &str = concatcp!(TOPIC_CONTROLLER_NAME, ".", AGENT_NAME); -const FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; -const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; - -/// Set by `kafka-reassign-partitions` and Cruise Control for the duration of a reassignment. -const THROTTLE_CONFIG_KEYS: [&str; 2] = [ - "leader.replication.throttled.replicas", - "follower.replication.throttled.replicas", -]; const RESYNC_INTERVAL: Duration = Duration::from_secs(5 * 60); const RETRY_INTERVAL: Duration = Duration::from_secs(10); const ERROR_RETRY_INTERVAL: Duration = Duration::from_secs(30); #[derive(Debug, Snafu, EnumDiscriminants)] +#[snafu(visibility(pub(crate)))] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { #[snafu(display("the KafkaTopic is invalid"))] @@ -67,14 +45,12 @@ pub enum Error { #[snafu(display("failed to read the bootstrap servers"))] ReadBootstrapServers { source: crate::config::Error }, + #[snafu(display("the KafkaCluster has no bootstrap servers yet"))] + NoBootstrapServers, + #[snafu(display("failed to talk to Kafka"))] Kafka { source: client::Error }, - #[snafu(display("failed to annotate the KafkaTopic"))] - Annotate { - source: stackable_operator::client::Error, - }, - #[snafu(display("failed to update the status of the KafkaTopic"))] UpdateStatus { source: stackable_operator::client::Error, @@ -86,7 +62,7 @@ pub enum Error { }, } -type Result = std::result::Result; +pub(crate) type Result = std::result::Result; impl ReconcilerError for Error { fn category(&self) -> &'static str { @@ -95,113 +71,92 @@ impl ReconcilerError for Error { } pub struct Ctx { - client: stackable_operator::client::Client, - cluster_name: ClusterName, - security: Security, - topics: Store>, + pub(crate) client: stackable_operator::client::Client, + pub(crate) cluster_name: ClusterName, + pub(crate) security: Security, + pub(crate) kafka_topics: Store>, /// One client per agent, rebuilt when the bootstrap servers change - kafka: Mutex>, + pub(crate) client_cache: Mutex>, } impl Ctx { - /// `topics` is the store of the controller, used to detect conflicts between `KafkaTopic`s. + /// `kafka_topics` is the store of the controller, used to detect conflicts between + /// `KafkaTopic`s. pub fn new( client: stackable_operator::client::Client, cluster_name: ClusterName, security: Security, - topics: Store>, + kafka_topics: Store>, ) -> Self { Self { client, cluster_name, security, - topics, - kafka: Mutex::new(None), + kafka_topics, + client_cache: Mutex::new(None), } } } -struct CachedClient { - bootstrap_servers: Vec, - client: KafkaAdminClient, -} - pub async fn reconcile_topic( - topic: Arc>, + kafka_topic: Arc>, ctx: Arc, ) -> Result { info!("Starting reconcile"); // dereference - let topic = topic + let kafka_topic = kafka_topic .0 .as_ref() .map_err(error_boundary::InvalidObject::clone) .context(InvalidKafkaTopicSnafu)?; - if topic.spec.cluster_name != ctx.cluster_name { + if kafka_topic.spec.cluster_ref.name != ctx.cluster_name { return Ok(Action::await_change()); } // finalizer (Kubernetes) - if let Some(action) = reconcile_finalizer(topic, &ctx).await? { + // Left untouched while paused, like operators that only update the status then. A deletion + // waits until the reconciliation is resumed. + let paused = kafka_topic.spec.topic_operation.reconciliation_paused; + if !paused && let Some(action) = reconcile_finalizer(kafka_topic, &ctx).await? { return Ok(action); } - // sync (Kafka + Kubernetes) - let (outcome, observed) = match sync(topic, &ctx).await { - Ok(synced) => synced, - Err(Error::Kafka { source }) => { - let outcome = TopicSyncOutcome::from_client_error(&source); - update_status(topic, &ctx, &outcome, None).await?; - return Err(Error::Kafka { source }); - } - Err(error) => return Err(error), - }; + // sync (Kafka) + let mut topic_state = None; + let sync_result = sync(kafka_topic, &ctx, &mut topic_state).await; - // update status (Kubernetes) - update_status(topic, &ctx, &outcome, observed.as_ref()).await?; - Ok(requeue(&outcome, observed.is_some())) -} - -/// Returns `Some` if the reconcile ends here. -async fn reconcile_finalizer(topic: &KafkaTopic, ctx: &Ctx) -> Result> { - let has_finalizer = topic - .finalizers() - .iter() - .any(|finalizer| finalizer == FINALIZER); - if topic.metadata.deletion_timestamp.is_some() { - if has_finalizer { - cleanup(topic, ctx).await?; - set_finalizer(topic, ctx, false).await?; - } - return Ok(Some(Action::await_change())); + // finalizer (Kubernetes) + // Added before the status, because the status patch changes the resourceVersion. + if !paused { + add_finalizer(kafka_topic, &ctx, topic_state.as_ref()).await?; } - let needs_finalizer = topic.spec.management_policy.deletes_on_cleanup(); - if needs_finalizer != has_finalizer { - // The patch triggers another reconcile with the updated object. - set_finalizer(topic, ctx, needs_finalizer).await?; - return Ok(Some(Action::await_change())); + // update status (Kubernetes) + match sync_result { + Ok(sync_outcome) => { + update_status(kafka_topic, &ctx, &sync_outcome, topic_state.as_ref()).await?; + Ok(requeue(&sync_outcome)) + } + Err(Error::Kafka { source }) => { + let sync_outcome = TopicSyncOutcome::from_client_error(&source); + update_status(kafka_topic, &ctx, &sync_outcome, topic_state.as_ref()).await?; + Err(Error::Kafka { source }) + } + Err(error) => Err(error), } - Ok(None) } -fn requeue(outcome: &TopicSyncOutcome, observed: bool) -> Action { - match outcome { +fn requeue(sync_outcome: &TopicSyncOutcome) -> Action { + match sync_outcome { TopicSyncOutcome::InvalidSpec { .. } => Action::await_change(), - // After a create or a grow, the next reconcile fills in the topic id and partitions. - TopicSyncOutcome::Synced { .. } | TopicSyncOutcome::UnsupportedChange { .. } - if !observed => - { - Action::requeue(RETRY_INTERVAL) - } TopicSyncOutcome::ClusterNotReady => Action::requeue(RETRY_INTERVAL), _ => Action::requeue(RESYNC_INTERVAL), } } pub fn error_policy( - _topic: Arc>, + _kafka_topic: Arc>, error: &Error, _ctx: Arc, ) -> Action { @@ -211,492 +166,19 @@ pub fn error_policy( } } -/// Returns the outcome and, if the `KafkaTopic` owns the topic, the described topic. After a -/// partition grow, the topic is left out, because the description is outdated. -async fn sync(topic: &KafkaTopic, ctx: &Ctx) -> Result<(TopicSyncOutcome, Option)> { - let topic_name = topic.spec.topic_name.as_ref(); - - // validate (no client required) - if let Some(outcome) = validate(topic, &ctx.topics) { - return Ok((outcome, None)); - } - - // observe (Kafka) - let Some(bootstrap_servers) = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) - .context(ReadBootstrapServersSnafu)? - else { - return Ok((TopicSyncOutcome::ClusterNotReady, None)); - }; - let mut cache = ctx.kafka.lock().await; - let kafka = client_for(&mut cache, bootstrap_servers, &ctx.security); - let observed = observe(kafka, topic_name).await?; - - // claim ownership (Kubernetes) - let claimed = if is_owned(&topic.metadata) { - false - } else { - if observed.is_some() && !topic.spec.management_policy.adopts_existing() { - let outcome = TopicSyncOutcome::AlreadyExists { - message: format!( - "the topic {topic_name:?} already exists in Kafka; set \ - managementPolicy.adoption to Adopt to take it over" - ), - }; - return Ok((outcome, None)); - } - claim_ownership(topic, ctx).await?; - true - }; - - // plan (no client required) - let recorded_keys = managed_config_keys(topic); - let topic_plan = plan(topic, observed.as_ref(), &recorded_keys); - let grows_partitions = matches!( - &topic_plan, - TopicPlan::Update(update) if update.grow_partitions_to.is_some() - ); - - // execute (Kafka + Kubernetes) - let outcome = match execute(topic, ctx, kafka, topic_plan, recorded_keys).await { - // The topic was created by someone else after it was observed. - Err(Error::Kafka { source }) if claimed && source.is_topic_already_exists() => { - release_ownership(topic, ctx).await?; - return Err(Error::Kafka { source }); - } - result => result?, - }; - - let observed = observed - .filter(|_| !grows_partitions) - .map(|observed| observed.topic); - Ok((outcome, observed)) -} - -/// Returns why the `KafkaTopic` cannot be synced, if it cannot. -fn validate( - topic: &KafkaTopic, - topics: &Store>, -) -> Option { - let throttle_keys = THROTTLE_CONFIG_KEYS - .iter() - .filter(|key| topic.spec.config.contains_key(**key)) - .copied() - .collect::>(); - if !throttle_keys.is_empty() { - return Some(TopicSyncOutcome::InvalidSpec { - message: format!( - "the config keys {throttle_keys:?} are managed by partition reassignments and \ - cannot be set" - ), - }); - } - conflicting_owner(topic, topics).map(|owner| TopicSyncOutcome::Conflict { - message: format!( - "the topic {:?} is managed by the KafkaTopic {owner:?}", - topic.spec.topic_name.as_ref() - ), - }) -} - -async fn observe(kafka: &mut KafkaAdminClient, topic_name: &str) -> Result> { - let Some(topic) = kafka.describe_topic(topic_name).await.context(KafkaSnafu)? else { - return Ok(None); - }; - let partitions = topic - .partitions - .iter() - .map(|partition| partition.index) - .collect::>(); - Ok(Some(ObservedTopic { - configs: kafka - .describe_topic_configs(topic_name) - .await - .context(KafkaSnafu)?, - reassignments: kafka - .list_partition_reassignments(topic_name, &partitions) - .await - .context(KafkaSnafu)?, - topic, - })) -} - -async fn execute( - topic: &KafkaTopic, - ctx: &Ctx, - kafka: &mut KafkaAdminClient, - topic_plan: TopicPlan, - mut recorded_keys: BTreeSet, -) -> Result { - let topic_name = topic.spec.topic_name.as_ref(); - let spec_keys = topic.spec.config.keys().cloned().collect::>(); - - let changes_configs = match &topic_plan { - TopicPlan::Create { .. } => true, - TopicPlan::Update(update) => { - !update.set_configs.is_empty() || !update.delete_configs.is_empty() - } - }; - let intended_keys = &recorded_keys | &spec_keys; - if changes_configs && intended_keys != recorded_keys { - record_managed_config_keys(topic, ctx, &intended_keys).await?; - recorded_keys = intended_keys; - } - - let outcome = match topic_plan { - TopicPlan::Create { - partitions, - replication_factor, - configs, - } => { - kafka - .create_topic(topic_name, partitions, replication_factor, &configs) - .await - .context(KafkaSnafu)?; - info!( - topic = topic_name, - partitions, - replication_factor, - ?configs, - kafka_topic = %topic.name_any(), - "Created the topic" - ); - TopicSyncOutcome::Synced { - partitions: partitions as usize, - reassigning_partitions: Vec::new(), - } - } - TopicPlan::Update(update) => { - if let Some(partitions) = update.grow_partitions_to { - kafka - .create_partitions(topic_name, partitions) - .await - .context(KafkaSnafu)?; - info!( - topic = topic_name, - partitions, - kafka_topic = %topic.name_any(), - "Increased the number of partitions" - ); - } - if changes_configs { - kafka - .incremental_alter_configs( - topic_name, - &update.set_configs, - &update.delete_configs, - ) - .await - .context(KafkaSnafu)?; - info!( - topic = topic_name, - set = ?update.set_configs, - reset = ?update.delete_configs, - kafka_topic = %topic.name_any(), - "Changed the topic configs" - ); - } - if update.unsupported.is_empty() { - TopicSyncOutcome::Synced { - partitions: topic.spec.partitions as usize, - reassigning_partitions: update.reassigning_partitions, - } - } else { - TopicSyncOutcome::UnsupportedChange { - messages: update.unsupported, - } - } - } - }; - - if recorded_keys != spec_keys { - record_managed_config_keys(topic, ctx, &spec_keys).await?; - } - - Ok(outcome) -} - -async fn cleanup(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { - if topic.spec.management_policy.deletes_on_cleanup() - && is_owned(&topic.metadata) - && conflicting_owner(topic, &ctx.topics).is_none() - { - let bootstrap_servers = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) - .context(ReadBootstrapServersSnafu)? - .unwrap_or_default(); - let mut cache = ctx.kafka.lock().await; - client_for(&mut cache, bootstrap_servers, &ctx.security) - .delete_topic(topic.spec.topic_name.as_ref()) - .await - .context(KafkaSnafu)?; - info!( - topic = topic.spec.topic_name.as_ref(), - kafka_topic = %topic.name_any(), - "Deleted the topic" - ); - } - Ok(()) -} - -/// Fails if the `KafkaTopic` was changed since it was read, so concurrent changes to the -/// finalizers are not lost. -async fn set_finalizer(topic: &KafkaTopic, ctx: &Ctx, present: bool) -> Result<()> { - let mut finalizers = topic - .finalizers() - .iter() - .filter(|finalizer| *finalizer != FINALIZER) - .cloned() - .collect::>(); - if present { - finalizers.push(FINALIZER.to_owned()); - } - let patch = json!({ - "metadata": { - "resourceVersion": topic.metadata.resource_version, - "finalizers": finalizers, - } - }); - ctx.client - .merge_patch(topic, patch) - .await - .context(UpdateFinalizersSnafu)?; - Ok(()) -} - -/// Returns the cached client, rebuilt if the bootstrap servers changed. -fn client_for<'a>( - cache: &'a mut Option, - bootstrap_servers: Vec, - security: &Security, -) -> &'a mut KafkaAdminClient { - if cache - .as_ref() - .is_some_and(|cached| cached.bootstrap_servers != bootstrap_servers) - { - *cache = None; - } - let cached = cache.get_or_insert_with(|| { - info!(?bootstrap_servers, "Connecting to new bootstrap servers"); - CachedClient { - bootstrap_servers: bootstrap_servers.clone(), - client: KafkaAdminClient::new(bootstrap_servers, security.clone(), KAFKA_CLIENT_ID), - } - }); - &mut cached.client -} - -/// The oldest other `KafkaTopic` for the same topic of the same cluster, by creation time and -/// name. Checked against all `KafkaTopic`s in the store, so the result does not depend on the -/// reconcile order. -fn conflicting_owner( - topic: &KafkaTopic, - topics: &Store>, -) -> Option { - let key = |topic: &KafkaTopic| (topic.metadata.creation_timestamp.clone(), topic.name_any()); - topics - .state() - .iter() - .filter_map(|other| other.0.as_ref().ok()) - .filter(|other| { - other.spec.cluster_name == topic.spec.cluster_name - && other.spec.topic_name == topic.spec.topic_name - }) - .min_by_key(|other| key(other)) - .filter(|oldest| key(oldest) != key(topic)) - .map(ResourceExt::name_any) -} - -/// Fails if the `KafkaTopic` was changed since it was read, so ownership is never claimed twice. -async fn claim_ownership(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { - let patch = json!({ - "metadata": { - "resourceVersion": topic.metadata.resource_version, - "annotations": { OWNED_SINCE_ANNOTATION: Timestamp::now().to_string() }, - } - }); - ctx.client - .merge_patch(topic, patch) - .await - .context(AnnotateSnafu)?; - Ok(()) -} - -async fn release_ownership(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { - let patch = json!({ - "metadata": { - "annotations": { - OWNED_SINCE_ANNOTATION: null, - MANAGED_CONFIG_KEYS_ANNOTATION: null, - }, - } - }); - ctx.client - .merge_patch(topic, patch) - .await - .context(AnnotateSnafu)?; - Ok(()) -} - -async fn record_managed_config_keys( - topic: &KafkaTopic, - ctx: &Ctx, - keys: &BTreeSet, -) -> Result<()> { - let patch = json!({ - "metadata": { "annotations": { MANAGED_CONFIG_KEYS_ANNOTATION: render(keys) } } - }); - ctx.client - .merge_patch(topic, patch) - .await - .context(AnnotateSnafu)?; - Ok(()) -} - -/// Patches the status only if it changed, so the agent does not trigger its own watch. -async fn update_status( - topic: &KafkaTopic, - ctx: &Ctx, - outcome: &TopicSyncOutcome, - observed: Option<&Topic>, -) -> Result<()> { - let current = topic.status.clone().unwrap_or_default(); - let mut status = KafkaTopicStatus { - conditions: update_conditions( - ¤t.conditions, - conditions(outcome), - &Time(Timestamp::now()), - ), - ..current.clone() - }; - match (outcome, observed) { - (_, Some(observed)) => { - status.topic_id = Some(observed.id.clone()); - status.partitions = observed - .partitions - .iter() - .map(|partition| KafkaTopicPartitionStatus { - index: partition.index, - replicas: partition.replicas.clone(), - }) - .collect(); - } - (TopicSyncOutcome::AlreadyExists { .. } | TopicSyncOutcome::Conflict { .. }, None) => { - status.topic_id = None; - status.partitions = Vec::new(); - } - _ => {} - } - - if status != current { - ctx.client - .apply_patch_status(AGENT_NAME, topic, &status) - .await - .context(UpdateStatusSnafu)?; - } - Ok(()) -} - #[cfg(test)] mod tests { - use stackable_operator::{ - k8s_openapi::apimachinery::pkg::apis::meta::v1::Time, kube::runtime::watcher, - }; - use super::*; - use crate::topic::test_support::kafka_topic; - - fn created_topic( - name: &str, - created_at: i64, - topic_name: &str, - ) -> DeserializeGuard { - let mut topic = kafka_topic(name, topic_name); - topic.metadata.creation_timestamp = Some(Time( - Timestamp::from_second(created_at).expect("the timestamp should be valid"), - )); - DeserializeGuard(Ok(topic)) - } - - fn store(topics: Vec>) -> Store> { - let (store, mut writer) = stackable_operator::kube::runtime::reflector::store(); - for topic in topics { - writer.apply_watcher_event(&watcher::Event::Apply(topic)); - } - store - } - - fn inner(topic: &DeserializeGuard) -> &KafkaTopic { - topic.0.as_ref().expect("the topic should be valid") - } - - #[test] - fn the_oldest_kafka_topic_owns_the_topic() { - let older = created_topic("older", 10, "orders"); - let newer = created_topic("newer", 20, "orders"); - let other_topic = created_topic("other", 5, "payments"); - let topics = store(vec![older.clone(), newer.clone(), other_topic]); - - assert_eq!(None, conflicting_owner(inner(&older), &topics)); - assert_eq!( - Some("older".to_owned()), - conflicting_owner(inner(&newer), &topics) - ); - } - - #[test] - fn the_name_decides_between_kafka_topics_of_the_same_age() { - let first = created_topic("a", 10, "orders"); - let second = created_topic("b", 10, "orders"); - let topics = store(vec![second.clone(), first.clone()]); - - assert_eq!(None, conflicting_owner(inner(&first), &topics)); - assert_eq!( - Some("a".to_owned()), - conflicting_owner(inner(&second), &topics) - ); - } - - #[rstest::rstest] - #[case::no_throttle(&[("retention.ms", "1000")], false)] - #[case::throttle(&[("follower.replication.throttled.replicas", "*")], true)] - fn reject_throttle_config_keys(#[case] config: &[(&str, &str)], #[case] invalid: bool) { - let mut topic = kafka_topic("orders", "orders"); - topic.spec.config = config - .iter() - .map(|(key, value)| (key.to_string(), value.to_string())) - .collect(); - - assert_eq!( - invalid, - matches!( - validate(&topic, &store(vec![])), - Some(TopicSyncOutcome::InvalidSpec { .. }) - ) - ); - } - - fn synced() -> TopicSyncOutcome { - TopicSyncOutcome::Synced { - partitions: 1, - reassigning_partitions: vec![], - } - } #[rstest::rstest] - #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: String::new() }, false, Action::await_change())] - #[case::created(synced(), false, Action::requeue(RETRY_INTERVAL))] - #[case::cluster_not_ready( - TopicSyncOutcome::ClusterNotReady, - false, - Action::requeue(RETRY_INTERVAL) + #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: String::new() }, Action::await_change())] + #[case::cluster_not_ready(TopicSyncOutcome::ClusterNotReady, Action::requeue(RETRY_INTERVAL))] + #[case::synced(TopicSyncOutcome::Synced { reassigning_partition_count: 0 }, Action::requeue(RESYNC_INTERVAL))] + #[case::paused( + TopicSyncOutcome::ReconciliationPaused, + Action::requeue(RESYNC_INTERVAL) )] - #[case::synced(synced(), true, Action::requeue(RESYNC_INTERVAL))] - #[case::grown_with_unsupported_change(TopicSyncOutcome::UnsupportedChange { messages: vec![] }, false, Action::requeue(RETRY_INTERVAL))] - #[case::unsupported_change(TopicSyncOutcome::UnsupportedChange { messages: vec![] }, true, Action::requeue(RESYNC_INTERVAL))] - fn requeue_by_outcome( - #[case] outcome: TopicSyncOutcome, - #[case] observed: bool, - #[case] expected: Action, - ) { - assert_eq!(expected, requeue(&outcome, observed)); + fn requeue_by_outcome(#[case] sync_outcome: TopicSyncOutcome, #[case] expected: Action) { + assert_eq!(expected, requeue(&sync_outcome)); } } diff --git a/rust/agent-binary/src/topic/finalizer.rs b/rust/agent-binary/src/topic/finalizer.rs new file mode 100644 index 00000000..78c4d4d3 --- /dev/null +++ b/rust/agent-binary/src/topic/finalizer.rs @@ -0,0 +1,137 @@ +//! The finalizer of a `KafkaTopic` and the cleanup in Kafka when it is deleted. + +use std::path::Path; + +use serde_json::json; +use snafu::{OptionExt, ResultExt}; +use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; +use stackable_operator::{ + crd::management_access::v1alpha1::DeletionPolicy, + kube::{ResourceExt, runtime::controller::Action}, +}; +use tracing::info; + +use super::{ + controller::{ + Ctx, KafkaSnafu, NoBootstrapServersSnafu, ReadBootstrapServersSnafu, Result, + UpdateFinalizersSnafu, + }, + plan::TopicState, + sync::client_for, + validate::has_other_kafka_topic, +}; +use crate::config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}; + +const FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; + +/// Handles a deletion and removes a finalizer that is no longer needed. Returns `Some` if the +/// reconcile ends here. +pub(crate) async fn reconcile_finalizer( + kafka_topic: &KafkaTopic, + ctx: &Ctx, +) -> Result> { + let has_finalizer = has_finalizer(kafka_topic); + + if kafka_topic.metadata.deletion_timestamp.is_some() { + if has_finalizer { + cleanup(kafka_topic, ctx).await?; + set_finalizer(kafka_topic, ctx, false).await?; + } + return Ok(Some(Action::await_change())); + } + + if has_finalizer && kafka_topic.spec.deletion_policy != DeletionPolicy::Delete { + // The patch triggers another reconcile with the updated object. + set_finalizer(kafka_topic, ctx, false).await?; + return Ok(Some(Action::await_change())); + } + Ok(None) +} + +/// Adds the finalizer only once the topic exists and is managed by this `KafkaTopic`, so deleting +/// a `KafkaTopic` that never managed the topic, e.g. because of an invalid spec or a conflict, +/// does not delete it. +pub(crate) async fn add_finalizer( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + topic_state: Option<&TopicState>, +) -> Result<()> { + if kafka_topic.spec.deletion_policy == DeletionPolicy::Delete + && topic_state.is_some() + && !has_finalizer(kafka_topic) + { + set_finalizer(kafka_topic, ctx, true).await?; + } + Ok(()) +} + +fn has_finalizer(kafka_topic: &KafkaTopic) -> bool { + kafka_topic + .finalizers() + .iter() + .any(|finalizer| finalizer == FINALIZER) +} + +/// Deletes the topic in Kafka if the `deletionPolicy` says so and no other `KafkaTopic` declares +/// it. +/// +/// The policy is read from the `KafkaTopic` being deleted, so switching it to `Retain` releases a +/// deletion that is stuck because Kafka cannot be reached. +async fn cleanup(kafka_topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + if kafka_topic.spec.deletion_policy != DeletionPolicy::Delete { + return Ok(()); + } + // A topic with an invalid name was never created. + let Ok(topic_name) = kafka_topic.topic_name() else { + return Ok(()); + }; + if has_other_kafka_topic(kafka_topic, &topic_name, &ctx.kafka_topics) { + return Ok(()); + } + + let bootstrap_servers = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) + .context(ReadBootstrapServersSnafu)? + .context(NoBootstrapServersSnafu)?; + let mut client_cache = ctx.client_cache.lock().await; + client_for(&mut client_cache, bootstrap_servers, &ctx.security) + .delete_topic(topic_name.as_ref()) + .await + .context(KafkaSnafu)?; + + info!( + topic = %topic_name, + kafka_topic = %kafka_topic.name_any(), + "Deleted the topic" + ); + + Ok(()) +} + +/// Fails if the `KafkaTopic` was changed since it was read, so concurrent changes to the +/// finalizers are not lost. +async fn set_finalizer( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + should_have_finalizer: bool, +) -> Result<()> { + let mut finalizers = kafka_topic + .finalizers() + .iter() + .filter(|finalizer| *finalizer != FINALIZER) + .cloned() + .collect::>(); + if should_have_finalizer { + finalizers.push(FINALIZER.to_owned()); + } + let patch = json!({ + "metadata": { + "resourceVersion": kafka_topic.metadata.resource_version, + "finalizers": finalizers, + } + }); + ctx.client + .merge_patch(kafka_topic, patch) + .await + .context(UpdateFinalizersSnafu)?; + Ok(()) +} diff --git a/rust/agent-binary/src/topic/managed_config_keys.rs b/rust/agent-binary/src/topic/managed_config_keys.rs deleted file mode 100644 index 069d3230..00000000 --- a/rust/agent-binary/src/topic/managed_config_keys.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! The topic config keys the agent manages, recorded on the `KafkaTopic`. -//! -//! Keys removed from the spec are reset in Kafka, but only if the agent set them before. Kafka -//! only accepts its known topic config names, none of which contain a comma, so the keys are -//! stored comma-separated. - -use std::collections::BTreeSet; - -use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; -use stackable_operator::kube::ResourceExt; - -pub const MANAGED_CONFIG_KEYS_ANNOTATION: &str = "kafka.stackable.tech/managed-config-keys"; - -pub fn managed_config_keys(topic: &KafkaTopic) -> BTreeSet { - topic - .annotations() - .get(MANAGED_CONFIG_KEYS_ANNOTATION) - .map(|value| { - value - .split(',') - .map(str::trim) - .filter(|key| !key.is_empty()) - .map(str::to_owned) - .collect() - }) - .unwrap_or_default() -} - -pub fn render(keys: &BTreeSet) -> String { - keys.iter() - .map(String::as_str) - .collect::>() - .join(",") -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use rstest::rstest; - - use super::*; - use crate::topic::test_support::kafka_topic; - - #[rstest] - #[case::no_annotation(None, &[])] - #[case::empty(Some(""), &[])] - #[case::keys(Some("retention.ms,cleanup.policy"), &["cleanup.policy", "retention.ms"])] - #[case::spaces_and_empty_entries(Some(" retention.ms, ,"), &["retention.ms"])] - fn read_managed_config_keys(#[case] annotation: Option<&str>, #[case] expected: &[&str]) { - let mut topic = kafka_topic("orders", "orders"); - topic.metadata.annotations = annotation.map(|value| { - BTreeMap::from([(MANAGED_CONFIG_KEYS_ANNOTATION.to_owned(), value.to_owned())]) - }); - - let expected = expected - .iter() - .map(|key| key.to_string()) - .collect::>(); - assert_eq!(expected, managed_config_keys(&topic)); - } - - #[test] - fn render_sorted_keys() { - let keys = BTreeSet::from(["retention.ms".to_owned(), "cleanup.policy".to_owned()]); - - assert_eq!("cleanup.policy,retention.ms", render(&keys)); - } -} diff --git a/rust/agent-binary/src/topic/mod.rs b/rust/agent-binary/src/topic/mod.rs index a1ef9ffd..062a1e7a 100644 --- a/rust/agent-binary/src/topic/mod.rs +++ b/rust/agent-binary/src/topic/mod.rs @@ -2,7 +2,10 @@ mod condition; pub mod controller; -mod managed_config_keys; +mod finalizer; mod plan; +mod sync; #[cfg(test)] mod test_support; +mod update_status; +mod validate; diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs index 44e5f71f..81e4ce02 100644 --- a/rust/agent-binary/src/topic/plan.rs +++ b/rust/agent-binary/src/topic/plan.rs @@ -2,95 +2,201 @@ use std::collections::{BTreeMap, BTreeSet}; -use stackable_kafka_admin::client::{PartitionReassignment, Topic}; -use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; +use stackable_kafka_admin::client::{ + ConfigSource, CreatedTopic, PartitionReassignment, Topic, TopicConfig, +}; +use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopicSpec}; + +/// Set by `kafka-reassign-partitions` and Cruise Control for the duration of a reassignment. +/// Resetting them would lift the throttle of a running reassignment. +pub const THROTTLE_CONFIG_KEYS: [&str; 2] = [ + "leader.replication.throttled.replicas", + "follower.replication.throttled.replicas", +]; /// The state of an existing topic in Kafka. pub struct ObservedTopic { pub topic: Topic, - pub configs: BTreeMap, + pub effective_configs: BTreeMap, pub reassignments: Vec, } +/// The state of a topic in Kafka as shown in the status. +#[derive(Debug, Eq, PartialEq)] +pub struct TopicState { + pub name: TopicName, + pub id: String, + pub partition_count: i32, + pub replication_factor: Option, + pub effective_configs: BTreeMap, +} + +impl TopicState { + pub fn observed(name: TopicName, observed: ObservedTopic) -> Self { + let replication_factor = replication_factor(&observed); + Self { + name, + id: observed.topic.id, + partition_count: observed.topic.partitions.len() as i32, + replication_factor, + effective_configs: observed.effective_configs, + } + } + + pub fn created(name: TopicName, created: CreatedTopic) -> Self { + Self { + name, + id: created.id, + partition_count: created.partitions, + replication_factor: Some(created.replication_factor), + effective_configs: created.configs, + } + } +} + #[derive(Debug, Eq, PartialEq)] pub enum TopicPlan { Create { - partitions: i32, - replication_factor: i16, - configs: BTreeMap, + partition_count: Option, + replication_factor: Option, + config_overrides: BTreeMap, + }, + Update { + topic_update: TopicUpdate, + /// Applied are only the changes in `topic_update` + unsupported_changes: Vec, }, - Update(TopicUpdate), } -/// Changes to an existing topic. No grow, configs or unsupported differences means the topic -/// matches the spec. +/// Changes to an existing topic. No changes means the topic matches the spec, apart from +/// [`UnsupportedChange`]s. #[derive(Debug, Default, Eq, PartialEq)] pub struct TopicUpdate { - pub grow_partitions_to: Option, - pub set_configs: BTreeMap, - pub delete_configs: BTreeSet, - /// Differences the agent does not apply, as messages for the status - pub unsupported: Vec, - pub reassigning_partitions: Vec, + /// Only increased, see [`UnsupportedChange::DecreasePartitions`] + pub new_partition_count: Option, + pub configs_to_set: BTreeMap, + /// Topic-level configs to remove, so that the broker configs apply + pub configs_to_reset: BTreeSet, } -pub fn plan( - topic: &KafkaTopic, - observed: Option<&ObservedTopic>, - managed_config_keys: &BTreeSet, -) -> TopicPlan { - let spec = &topic.spec; +/// A difference to the spec the agent does not apply. The other changes are still applied. +#[derive(Debug, Eq, PartialEq)] +pub enum UnsupportedChange { + /// Kafka cannot remove partitions. + DecreasePartitions { desired: i32, actual: i32 }, + /// The partitions have between `actual_min` and `actual_max` replicas, not counting replicas + /// that a running reassignment removes. + /// + /// Kafka can only change the replicas of existing partitions with + /// `AlterPartitionReassignments` and explicit replica lists. Choosing the brokers is left to + /// `kafka-reassign-partitions` or Cruise Control. + ChangeReplicationFactor { + desired: i16, + actual_min: usize, + actual_max: usize, + }, +} + +pub fn plan(spec: &KafkaTopicSpec, observed: Option<&ObservedTopic>) -> TopicPlan { let Some(observed) = observed else { return TopicPlan::Create { - partitions: spec.partitions, + partition_count: spec.partitions, replication_factor: spec.replication_factor, - configs: spec.config.clone(), + config_overrides: spec.config_overrides.clone(), }; }; - let mut update = TopicUpdate::default(); - - let observed_partitions = observed.topic.partitions.len() as i32; - if spec.partitions > observed_partitions { - update.grow_partitions_to = Some(spec.partitions); - } else if spec.partitions < observed_partitions { - update.unsupported.push(format!( - "the topic has {observed_partitions} partitions, but the spec requests {}; partitions cannot be \ - removed", - spec.partitions - )); + let actual_partition_count = observed.topic.partitions.len() as i32; + let (configs_to_set, configs_to_reset) = + config_changes(&spec.config_overrides, &observed.effective_configs); + + TopicPlan::Update { + topic_update: TopicUpdate { + new_partition_count: spec + .partitions + .filter(|desired| *desired > actual_partition_count), + configs_to_set, + configs_to_reset, + }, + unsupported_changes: unsupported_changes(spec, observed), + } +} + +fn unsupported_changes(spec: &KafkaTopicSpec, observed: &ObservedTopic) -> Vec { + let mut changes = Vec::new(); + + let actual_partition_count = observed.topic.partitions.len() as i32; + if let Some(desired) = spec.partitions + && desired < actual_partition_count + { + changes.push(UnsupportedChange::DecreasePartitions { + desired, + actual: actual_partition_count, + }); } - if let Some(message) = replication_factor_difference( - spec.replication_factor, - &target_replication_factors(observed), - ) { - update.unsupported.push(message); + let replication_factors = target_replication_factors(observed); + if let Some(desired) = spec.replication_factor + && let (Some(&actual_min), Some(&actual_max)) = + (replication_factors.first(), replication_factors.last()) + && (actual_min != actual_max || actual_min != desired as usize) + { + changes.push(UnsupportedChange::ChangeReplicationFactor { + desired, + actual_min, + actual_max, + }); } - update.set_configs = spec - .config + changes +} + +/// The configs to set and the configs to reset, so that the topic-level configs are exactly the +/// desired ones. The throttles of partition reassignments are left alone. +fn config_changes( + desired: &BTreeMap, + effective_configs: &BTreeMap, +) -> (BTreeMap, BTreeSet) { + // Topic configs have no PASSWORD type, so Kafka does not redact their topic-level values. + let topic_level_configs = effective_configs + .iter() + .filter(|(_, config)| config.source == ConfigSource::DynamicTopic) + .map(|(key, config)| (key, config.value.as_deref())) + .collect::>(); + + // Compared with the topic-level value, not the effective one: a desired value that equals the + // broker config is still pinned on the topic. Otherwise a later change of the broker config + // would apply to the topic until the next resync. Unchanged values are skipped, so a resync + // without drift does not write to Kafka. + let configs_to_set = desired .iter() - .filter(|(key, value)| observed.configs.get(*key) != Some(*value)) + .filter(|(key, value)| topic_level_configs.get(key) != Some(&Some(value.as_str()))) .map(|(key, value)| (key.clone(), value.clone())) .collect(); - update.delete_configs = managed_config_keys - .iter() - .filter(|key| !spec.config.contains_key(*key) && observed.configs.contains_key(*key)) - .cloned() - .collect(); - - update.reassigning_partitions = observed - .reassignments - .iter() - .map(|reassignment| reassignment.index) + let configs_to_reset = topic_level_configs + .keys() + .filter(|key| !desired.contains_key(**key) && !THROTTLE_CONFIG_KEYS.contains(&key.as_str())) + .map(|key| key.to_string()) .collect(); + (configs_to_set, configs_to_reset) +} - TopicPlan::Update(update) +/// The replication factor of the partitions, excluding replicas a reassignment removes. `None` if +/// the partitions have different replication factors. +fn replication_factor(observed: &ObservedTopic) -> Option { + match target_replication_factors(observed).as_slice() { + [replication_factor] => i16::try_from(*replication_factor).ok(), + _ => None, + } } -/// The replication factor of each partition, excluding replicas a reassignment removes. -fn target_replication_factors(observed: &ObservedTopic) -> BTreeSet { +/// The distinct replication factors of the partitions in ascending order, excluding replicas a +/// reassignment removes. +/// +/// Kafka stores no replication factor per topic. During a reassignment, the replicas of a +/// partition include the ones being added and removed. `kafka-topics.sh --describe` shows the +/// replicas minus the ones being added of the first partition instead. +fn target_replication_factors(observed: &ObservedTopic) -> Vec { observed .topic .partitions @@ -100,35 +206,14 @@ fn target_replication_factors(observed: &ObservedTopic) -> BTreeSet { .reassignments .iter() .find(|reassignment| reassignment.index == partition.index) - .map_or(0, |reassignment| reassignment.removing_replicas.len()); - partition.replicas.len().saturating_sub(removing) + .map_or(0, |reassignment| reassignment.removing_replicas); + partition.replicas.saturating_sub(removing) }) + .collect::>() + .into_iter() .collect() } -fn replication_factor_difference( - desired: i16, - replication_factors: &BTreeSet, -) -> Option { - let (Some(min), Some(max)) = (replication_factors.first(), replication_factors.last()) else { - return None; - }; - - let hint = "change it with kafka-reassign-partitions"; - if min != max { - Some(format!( - "the partitions have between {min} and {max} replicas, but the spec requests a \ - replication factor of {desired}; {hint}" - )) - } else if *min != desired as usize { - Some(format!( - "the topic has a replication factor of {min}, but the spec requests {desired}; {hint}" - )) - } else { - None - } -} - #[cfg(test)] mod tests { use stackable_kafka_admin::client::Partition; @@ -136,12 +221,16 @@ mod tests { use super::*; use crate::topic::test_support::kafka_topic; - fn topic(partitions: i32, replication_factor: i16, config: &[(&str, &str)]) -> KafkaTopic { - let mut topic = kafka_topic("orders", "orders"); - topic.spec.partitions = partitions; - topic.spec.replication_factor = replication_factor; - topic.spec.config = configs(config); - topic + fn spec( + partitions: Option, + replication_factor: Option, + config: &[(&str, &str)], + ) -> KafkaTopicSpec { + let mut spec = kafka_topic("orders", Some("orders")).spec; + spec.partitions = partitions; + spec.replication_factor = replication_factor; + spec.config_overrides = configs(config); + spec } fn configs(entries: &[(&str, &str)]) -> BTreeMap { @@ -155,8 +244,30 @@ mod tests { keys.iter().map(|key| key.to_string()).collect() } - /// A topic whose partitions have the given replicas - fn observed(replicas: &[&[i32]], config: &[(&str, &str)]) -> ObservedTopic { + fn topic_configs(entries: &[(&str, &str, ConfigSource)]) -> BTreeMap { + entries + .iter() + .map(|(key, value, source)| { + ( + key.to_string(), + TopicConfig { + value: Some(value.to_string()), + source: *source, + }, + ) + }) + .collect() + } + + fn update(topic_update: TopicUpdate, unsupported_changes: Vec) -> TopicPlan { + TopicPlan::Update { + topic_update, + unsupported_changes, + } + } + + /// A topic whose partitions have the given number of replicas + fn observed(replicas: &[usize], configs: &[(&str, &str, ConfigSource)]) -> ObservedTopic { ObservedTopic { topic: Topic { id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), @@ -165,24 +276,24 @@ mod tests { .zip(0..) .map(|(replicas, index)| Partition { index, - replicas: replicas.to_vec(), + replicas: *replicas, }) .collect(), }, - configs: configs(config), + effective_configs: topic_configs(configs), reassignments: vec![], } } #[test] fn create_missing_topic() { - let plan = plan(&topic(3, 2, &[("retention.ms", "1000")]), None, &keys(&[])); + let plan = plan(&spec(Some(3), None, &[("retention.ms", "1000")]), None); assert_eq!( TopicPlan::Create { - partitions: 3, - replication_factor: 2, - configs: configs(&[("retention.ms", "1000")]), + partition_count: Some(3), + replication_factor: None, + config_overrides: configs(&[("retention.ms", "1000")]), }, plan ); @@ -190,135 +301,161 @@ mod tests { #[test] fn do_nothing_for_a_matching_topic() { - let observed = observed(&[&[1, 2], &[2, 3]], &[("retention.ms", "1000")]); + let observed = observed( + &[2, 2], + &[("retention.ms", "1000", ConfigSource::DynamicTopic)], + ); + let spec = spec(Some(2), Some(2), &[("retention.ms", "1000")]); - let plan = plan( - &topic(2, 2, &[("retention.ms", "1000")]), - Some(&observed), - &keys(&["retention.ms"]), + assert_eq!( + update(TopicUpdate::default(), vec![]), + plan(&spec, Some(&observed)) ); + } - assert_eq!(TopicPlan::Update(TopicUpdate::default()), plan); + #[test] + fn leave_partitions_and_replication_factor_alone_if_not_set() { + let observed = observed(&[3, 3, 3], &[]); + let spec = spec(None, None, &[]); + + assert_eq!( + update(TopicUpdate::default(), vec![]), + plan(&spec, Some(&observed)) + ); } #[test] fn grow_partitions() { - let observed = observed(&[&[1], &[2]], &[]); + let observed = observed(&[1, 1], &[]); - let plan = plan(&topic(4, 1, &[]), Some(&observed), &keys(&[])); + let plan = plan(&spec(Some(4), Some(1), &[]), Some(&observed)); assert_eq!( - TopicPlan::Update(TopicUpdate { - grow_partitions_to: Some(4), - ..TopicUpdate::default() - }), + update( + TopicUpdate { + new_partition_count: Some(4), + ..TopicUpdate::default() + }, + vec![] + ), plan ); } #[test] fn report_fewer_partitions_as_unsupported() { - let observed = observed(&[&[1], &[2], &[3]], &[]); + let observed = observed(&[1, 1, 1], &[]); - let TopicPlan::Update(update) = plan(&topic(2, 1, &[]), Some(&observed), &keys(&[])) else { - panic!("the topic exists"); - }; + let plan = plan(&spec(Some(2), None, &[]), Some(&observed)); - assert_eq!(None, update.grow_partitions_to); - assert_eq!(1, update.unsupported.len()); - assert!(update.unsupported[0].contains("cannot be removed")); + assert_eq!( + update( + TopicUpdate::default(), + vec![UnsupportedChange::DecreasePartitions { + desired: 2, + actual: 3 + }] + ), + plan + ); } #[test] - fn set_changed_and_reset_removed_configs() { + fn replace_the_topic_level_configs() { let observed = observed( - &[&[1]], + &[1], &[ - ("retention.ms", "1000"), - ("cleanup.policy", "compact"), - ("segment.bytes", "1024"), - ("leader.replication.throttled.replicas", "0:1"), + ("retention.ms", "1000", ConfigSource::DynamicTopic), + ("cleanup.policy", "compact", ConfigSource::DynamicTopic), + ("segment.bytes", "1024", ConfigSource::Default), + ("max.message.bytes", "10", ConfigSource::StaticBroker), + ( + "leader.replication.throttled.replicas", + "0:1", + ConfigSource::DynamicTopic, + ), ], ); let plan = plan( - &topic( - 1, - 1, + &spec( + None, + None, &[("retention.ms", "2000"), ("max.message.bytes", "10")], ), Some(&observed), - &keys(&["retention.ms", "cleanup.policy", "min.insync.replicas"]), ); assert_eq!( - TopicPlan::Update(TopicUpdate { - set_configs: configs(&[("max.message.bytes", "10"), ("retention.ms", "2000")]), - // `segment.bytes` and the throttle were set by others, `min.insync.replicas` is - // already gone - delete_configs: keys(&["cleanup.policy"]), - ..TopicUpdate::default() - }), + update( + TopicUpdate { + // `max.message.bytes` already has the value, but not on the topic + configs_to_set: configs(&[ + ("max.message.bytes", "10"), + ("retention.ms", "2000") + ]), + // `segment.bytes` is not set on the topic, the throttle belongs to a + // reassignment + configs_to_reset: keys(&["cleanup.policy"]), + ..TopicUpdate::default() + }, + vec![] + ), plan ); } #[test] fn report_a_different_replication_factor_as_unsupported() { - let observed = observed(&[&[1, 2, 3], &[2, 3, 4]], &[]); + let observed = observed(&[3, 3], &[]); - let TopicPlan::Update(update) = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])) else { - panic!("the topic exists"); - }; - - assert_eq!(1, update.unsupported.len()); - assert!(update.unsupported[0].contains("replication factor of 3")); + assert_eq!( + vec![UnsupportedChange::ChangeReplicationFactor { + desired: 2, + actual_min: 3, + actual_max: 3 + }], + unsupported_changes(&spec(None, Some(2), &[]), &observed) + ); } #[test] fn report_uneven_replica_counts_as_unsupported() { - let observed = observed(&[&[1, 2], &[2, 3, 4]], &[]); + let observed = observed(&[2, 3], &[]); - let TopicPlan::Update(update) = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])) else { - panic!("the topic exists"); - }; - - assert_eq!(1, update.unsupported.len()); - assert!(update.unsupported[0].contains("between 2 and 3 replicas")); + assert_eq!( + vec![UnsupportedChange::ChangeReplicationFactor { + desired: 2, + actual_min: 2, + actual_max: 3 + }], + unsupported_changes(&spec(None, Some(2), &[]), &observed) + ); } #[test] fn accept_a_reassignment_towards_the_desired_replication_factor() { - let mut observed = observed(&[&[4, 5, 1, 2], &[2, 3]], &[]); + let mut observed = observed(&[4, 2], &[]); observed.reassignments = vec![PartitionReassignment { index: 0, - adding_replicas: vec![4, 5], - removing_replicas: vec![1, 2], + removing_replicas: 2, }]; - let plan = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])); - assert_eq!( - TopicPlan::Update(TopicUpdate { - reassigning_partitions: vec![0], - ..TopicUpdate::default() - }), - plan + Vec::::new(), + unsupported_changes(&spec(None, Some(2), &[]), &observed) ); } #[test] - fn exclude_removed_replicas_from_the_replication_factor() { - let mut observed = observed(&[&[4, 5, 1, 2], &[2, 3, 4]], &[]); - observed.reassignments = vec![PartitionReassignment { + fn report_the_target_replication_factor() { + let mut reassigning = observed(&[4, 2], &[]); + reassigning.reassignments = vec![PartitionReassignment { index: 0, - adding_replicas: vec![4, 5], - removing_replicas: vec![1, 2], + removing_replicas: 2, }]; - assert_eq!( - BTreeSet::from([2, 3]), - target_replication_factors(&observed) - ); + assert_eq!(Some(2), replication_factor(&reassigning)); + assert_eq!(None, replication_factor(&observed(&[2, 3], &[]))); } } diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs new file mode 100644 index 00000000..7608efe9 --- /dev/null +++ b/rust/agent-binary/src/topic/sync.rs @@ -0,0 +1,280 @@ +//! Brings the topic in Kafka to the state of its `KafkaTopic` spec. + +use std::{ + collections::{BTreeMap, BTreeSet}, + path::Path, +}; + +use snafu::ResultExt; +use stackable_kafka_admin::{ + bootstrap::BrokerAddress, + client::{KafkaAdminClient, TopicConfig}, + connection::Security, +}; +use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopic}; +use stackable_operator::kube::ResourceExt; +use tracing::info; + +use super::{ + condition::TopicSyncOutcome, + controller::{Ctx, KafkaSnafu, ReadBootstrapServersSnafu, Result}, + plan::{ObservedTopic, TopicPlan, TopicState, plan}, + validate::validate, +}; +use crate::config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}; + +const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; + +pub(crate) struct CachedClient { + bootstrap_servers: Vec, + admin_client: KafkaAdminClient, +} + +/// Sets `topic_state` to the topic in Kafka, if it exists and is managed by this `KafkaTopic`. +/// It is kept up to date with each applied change, also if a later step fails. +pub(crate) async fn sync( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + topic_state: &mut Option, +) -> Result { + // validate (no client required) + let topic_name = match validate(kafka_topic, &ctx.kafka_topics) { + Ok(topic_name) => topic_name, + Err(sync_outcome) => return Ok(sync_outcome), + }; + + // observe (Kafka) + let Some(bootstrap_servers) = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) + .context(ReadBootstrapServersSnafu)? + else { + return Ok(TopicSyncOutcome::ClusterNotReady); + }; + let mut client_cache = ctx.client_cache.lock().await; + let admin_client = client_for(&mut client_cache, bootstrap_servers, &ctx.security); + let observed = observe(admin_client, &topic_name).await?; + + // plan (no client required) + let topic_plan = plan(&kafka_topic.spec, observed.as_ref()); + let reassigning_partition_count = observed + .as_ref() + .map_or(0, |observed| observed.reassignments.len()); + *topic_state = observed.map(|observed| TopicState::observed(topic_name.clone(), observed)); + if kafka_topic.spec.topic_operation.reconciliation_paused { + return Ok(TopicSyncOutcome::ReconciliationPaused); + } + + // execute (Kafka) + execute( + kafka_topic, + admin_client, + topic_name, + &topic_plan, + topic_state, + ) + .await?; + + let sync_outcome = match topic_plan { + TopicPlan::Update { + unsupported_changes, + .. + } if !unsupported_changes.is_empty() => TopicSyncOutcome::UnsupportedChange { + changes: unsupported_changes, + }, + _ => TopicSyncOutcome::Synced { + reassigning_partition_count, + }, + }; + Ok(sync_outcome) +} + +async fn observe( + admin_client: &mut KafkaAdminClient, + topic_name: &TopicName, +) -> Result> { + let topic_name = topic_name.as_ref(); + let Some(topic) = admin_client + .describe_topic(topic_name) + .await + .context(KafkaSnafu)? + else { + return Ok(None); + }; + let partitions = topic + .partitions + .iter() + .map(|partition| partition.index) + .collect::>(); + Ok(Some(ObservedTopic { + effective_configs: admin_client + .describe_topic_configs(topic_name) + .await + .context(KafkaSnafu)?, + reassignments: admin_client + .list_partition_reassignments(topic_name, &partitions) + .await + .context(KafkaSnafu)?, + topic, + })) +} + +/// Applies the changes to `topic_state`, the topic in Kafka before them. +async fn execute( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: TopicName, + topic_plan: &TopicPlan, + topic_state: &mut Option, +) -> Result<()> { + match topic_plan { + TopicPlan::Create { + partition_count, + replication_factor, + config_overrides, + } => { + let created_state = create( + kafka_topic, + admin_client, + topic_name, + *partition_count, + *replication_factor, + config_overrides, + ) + .await?; + *topic_state = Some(created_state); + } + TopicPlan::Update { topic_update, .. } => { + // Partitions + if let Some(new_partition_count) = topic_update.new_partition_count { + grow_partitions(kafka_topic, admin_client, &topic_name, new_partition_count) + .await?; + if let Some(topic_state) = topic_state { + topic_state.partition_count = new_partition_count; + } + } + + // Config + if !topic_update.configs_to_set.is_empty() || !topic_update.configs_to_reset.is_empty() + { + let effective_configs = replace_configs( + kafka_topic, + admin_client, + &topic_name, + &topic_update.configs_to_set, + &topic_update.configs_to_reset, + ) + .await?; + if let Some(topic_state) = topic_state { + topic_state.effective_configs = effective_configs; + } + } + } + } + Ok(()) +} + +/// The CreateTopics response contains the topic id, the counts and the effective configs, so no +/// describe is needed afterwards. Without the `DESCRIBE_CONFIGS` permission, Kafka creates the +/// topic but the request fails, and so does every later sync, because observing the topic needs +/// that permission as well. +async fn create( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: TopicName, + partition_count: Option, + replication_factor: Option, + config_overrides: &BTreeMap, +) -> Result { + let created = admin_client + .create_topic( + topic_name.as_ref(), + partition_count, + replication_factor, + config_overrides, + ) + .await + .context(KafkaSnafu)?; + info!( + topic = %topic_name, + partitions = created.partitions, + replication_factor = created.replication_factor, + configs = ?config_overrides, + kafka_topic = %kafka_topic.name_any(), + "Created the topic" + ); + Ok(TopicState::created(topic_name, created)) +} + +async fn grow_partitions( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: &TopicName, + partition_count: i32, +) -> Result<()> { + admin_client + .create_partitions(topic_name.as_ref(), partition_count) + .await + .context(KafkaSnafu)?; + info!( + topic = %topic_name, + partitions = partition_count, + kafka_topic = %kafka_topic.name_any(), + "Increased the number of partitions" + ); + Ok(()) +} + +/// Returns the effective configs of the topic after the change. They are described again, +/// because IncrementalAlterConfigs does not return them and a reset key falls back to a broker +/// config the agent does not know. +async fn replace_configs( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: &TopicName, + configs_to_set: &BTreeMap, + configs_to_reset: &BTreeSet, +) -> Result> { + admin_client + .incremental_alter_configs(topic_name.as_ref(), configs_to_set, configs_to_reset) + .await + .context(KafkaSnafu)?; + info!( + topic = %topic_name, + set = ?configs_to_set, + reset = ?configs_to_reset, + kafka_topic = %kafka_topic.name_any(), + "Changed the topic configs" + ); + admin_client + .describe_topic_configs(topic_name.as_ref()) + .await + .context(KafkaSnafu) +} + +/// Returns the cached client, rebuilt if the bootstrap servers changed. +/// +/// The client reads the TLS files on every new connection and reconnects after a failed request, +/// so rotated certificates are picked up without restarting the agent. +pub(crate) fn client_for<'a>( + client_cache: &'a mut Option, + bootstrap_servers: Vec, + security: &Security, +) -> &'a mut KafkaAdminClient { + if client_cache + .as_ref() + .is_some_and(|cached| cached.bootstrap_servers != bootstrap_servers) + { + *client_cache = None; + } + let cached = client_cache.get_or_insert_with(|| { + info!(?bootstrap_servers, "Connecting to new bootstrap servers"); + CachedClient { + bootstrap_servers: bootstrap_servers.clone(), + admin_client: KafkaAdminClient::new( + bootstrap_servers, + security.clone(), + KAFKA_CLIENT_ID, + ), + } + }); + &mut cached.admin_client +} diff --git a/rust/agent-binary/src/topic/test_support.rs b/rust/agent-binary/src/topic/test_support.rs index 44830af9..bba58c0b 100644 --- a/rust/agent-binary/src/topic/test_support.rs +++ b/rust/agent-binary/src/topic/test_support.rs @@ -2,22 +2,27 @@ use std::str::FromStr; use stackable_kafka_shared::crd::topic::{ TopicName, - v1alpha1::{KafkaTopic, KafkaTopicSpec}, + v1alpha1::{KafkaClusterRef, KafkaTopic, KafkaTopicSpec}, }; use stackable_operator::v2::types::operator::ClusterName; /// A `KafkaTopic` of the cluster `simple-kafka` with one partition, one replica and no configs. -pub fn kafka_topic(name: &str, topic_name: &str) -> KafkaTopic { +pub fn kafka_topic(name: &str, topic_name: Option<&str>) -> KafkaTopic { KafkaTopic::new( name, KafkaTopicSpec { - cluster_name: ClusterName::from_str("simple-kafka") - .expect("the cluster name should be valid"), - management_policy: Default::default(), - topic_name: TopicName::from_str(topic_name).expect("the topic name should be valid"), - partitions: 1, - replication_factor: 1, - config: Default::default(), + cluster_ref: KafkaClusterRef { + name: ClusterName::from_str("simple-kafka") + .expect("the cluster name should be valid"), + }, + topic_name: topic_name.map(|topic_name| { + TopicName::from_str(topic_name).expect("the topic name should be valid") + }), + partitions: Some(1), + replication_factor: Some(1), + config_overrides: Default::default(), + deletion_policy: Default::default(), + topic_operation: Default::default(), }, ) } diff --git a/rust/agent-binary/src/topic/update_status.rs b/rust/agent-binary/src/topic/update_status.rs new file mode 100644 index 00000000..3c6ffd07 --- /dev/null +++ b/rust/agent-binary/src/topic/update_status.rs @@ -0,0 +1,196 @@ +//! The status of a `KafkaTopic` for the outcome of a sync. + +use snafu::ResultExt; +use stackable_kafka_shared::crd::topic::v1alpha1::{ + KafkaTopic, KafkaTopicConfigStatus, KafkaTopicStatus, +}; +use stackable_operator::{ + k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, + status::condition::managed_resource::update_conditions, +}; + +use super::{ + condition::{TopicSyncOutcome, conditions_from}, + controller::{Ctx, Result, UpdateStatusSnafu}, + plan::TopicState, +}; +use crate::AGENT_NAME; + +/// Patches the status only if it changed, so the agent does not trigger its own watch. +pub(crate) async fn update_status( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> Result<()> { + let current_status = kafka_topic.status.clone().unwrap_or_default(); + let new_status = status_from( + ¤t_status, + sync_outcome, + topic_state, + &Time(Timestamp::now()), + kafka_topic.metadata.generation, + ); + + if new_status != current_status { + ctx.client + .apply_patch_status(AGENT_NAME, kafka_topic, &new_status) + .await + .context(UpdateStatusSnafu)?; + } + Ok(()) +} + +/// Fields not known after the outcome are kept from `current_status`. +fn status_from( + current_status: &KafkaTopicStatus, + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, + now: &Time, + observed_generation: Option, +) -> KafkaTopicStatus { + let conditions = update_conditions( + ¤t_status.conditions, + conditions_from(sync_outcome, topic_state), + now, + observed_generation, + ); + match (sync_outcome, topic_state) { + (_, Some(topic_state)) => KafkaTopicStatus { + conditions, + topic_id: Some(topic_state.id.clone()), + topic_name: Some(topic_state.name.clone()), + partitions: Some(topic_state.partition_count), + replication_factor: topic_state.replication_factor, + config: topic_state + .effective_configs + .iter() + .map(|(key, config)| { + let config = KafkaTopicConfigStatus { + value: config.value.clone(), + source: config.source.to_string(), + }; + (key.clone(), config) + }) + .collect(), + }, + // The topic does not exist or is managed by another KafkaTopic. + (TopicSyncOutcome::Conflict { .. } | TopicSyncOutcome::ReconciliationPaused, None) => { + KafkaTopicStatus { + conditions, + ..KafkaTopicStatus::default() + } + } + _ => KafkaTopicStatus { + conditions, + ..current_status.clone() + }, + } +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, str::FromStr}; + + use stackable_kafka_admin::client::{ConfigSource, TopicConfig}; + use stackable_kafka_shared::crd::topic::TopicName; + + use super::*; + + fn now() -> Time { + Time(Timestamp::from_second(10).expect("the timestamp should be valid")) + } + + fn topic_state() -> TopicState { + TopicState { + name: TopicName::from_str("orders").expect("the topic name should be valid"), + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partition_count: 6, + replication_factor: Some(3), + effective_configs: BTreeMap::from([( + "retention.ms".to_owned(), + TopicConfig { + value: Some("1000".to_owned()), + source: ConfigSource::DynamicTopic, + }, + )]), + } + } + + fn synced() -> TopicSyncOutcome { + TopicSyncOutcome::Synced { + reassigning_partition_count: 0, + } + } + + #[test] + fn show_the_topic_in_kafka() { + let status = status_from( + &KafkaTopicStatus::default(), + &synced(), + Some(&topic_state()), + &now(), + Some(2), + ); + + assert_eq!(Some("UXyU9i5ARn6W00ON2taeWA".to_owned()), status.topic_id); + assert_eq!(Some(6), status.partitions); + assert_eq!(Some(3), status.replication_factor); + assert_eq!( + Some(&KafkaTopicConfigStatus { + value: Some("1000".to_owned()), + source: "DYNAMIC_TOPIC_CONFIG".to_owned(), + }), + status.config.get("retention.ms") + ); + assert!( + status + .conditions + .iter() + .all(|condition| condition.observed_generation == Some(2)) + ); + } + + #[test] + fn keep_the_topic_if_kafka_cannot_be_reached() { + let current = status_from( + &KafkaTopicStatus::default(), + &synced(), + Some(&topic_state()), + &now(), + Some(1), + ); + + let status = status_from( + ¤t, + &TopicSyncOutcome::ConnectionFailed { + message: "timed out".to_owned(), + }, + None, + &now(), + Some(1), + ); + + assert_eq!(current.topic_id, status.topic_id); + assert_eq!(current.config, status.config); + } + + #[rstest::rstest] + #[case::conflict(TopicSyncOutcome::Conflict { message: String::new() })] + #[case::paused_without_topic(TopicSyncOutcome::ReconciliationPaused)] + fn clear_the_topic_if_it_is_not_managed(#[case] sync_outcome: TopicSyncOutcome) { + let current = status_from( + &KafkaTopicStatus::default(), + &synced(), + Some(&topic_state()), + &now(), + Some(1), + ); + + let status = status_from(¤t, &sync_outcome, None, &now(), Some(1)); + + assert_eq!(None, status.topic_id); + assert!(status.config.is_empty()); + assert_eq!(2, status.conditions.len()); + } +} diff --git a/rust/agent-binary/src/topic/validate.rs b/rust/agent-binary/src/topic/validate.rs new file mode 100644 index 00000000..d5055d4d --- /dev/null +++ b/rust/agent-binary/src/topic/validate.rs @@ -0,0 +1,234 @@ +//! Whether a `KafkaTopic` can be synced, without talking to Kafka. + +use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopic}; +use stackable_operator::kube::{ResourceExt, core::DeserializeGuard, runtime::reflector::Store}; + +use super::{condition::TopicSyncOutcome, plan::THROTTLE_CONFIG_KEYS}; + +/// Returns the name of the topic in Kafka, or why the `KafkaTopic` cannot be synced. +pub(crate) fn validate( + kafka_topic: &KafkaTopic, + kafka_topics: &Store>, +) -> Result { + let topic_name = kafka_topic + .topic_name() + .map_err(|error| TopicSyncOutcome::InvalidSpec { + message: format!( + "metadata.name is not a valid topic name, recreate the KafkaTopic with topicName \ + set: {error}" + ), + })?; + + let throttle_keys = THROTTLE_CONFIG_KEYS + .iter() + .filter(|key| kafka_topic.spec.config_overrides.contains_key(**key)) + .copied() + .collect::>(); + if !throttle_keys.is_empty() { + return Err(TopicSyncOutcome::InvalidSpec { + message: format!( + "the config keys {throttle_keys:?} are managed by partition reassignments and \ + cannot be set" + ), + }); + } + + if let Some(owner) = conflicting_owner(kafka_topic, &topic_name, kafka_topics) { + return Err(TopicSyncOutcome::Conflict { + message: format!("the topic {topic_name} is managed by the KafkaTopic {owner:?}"), + }); + } + + Ok(topic_name) +} + +/// The oldest other `KafkaTopic` for the same topic of the same cluster, by creation time and +/// name. Checked against all `KafkaTopic`s in the store, so the result does not depend on the +/// reconcile order. +pub(crate) fn conflicting_owner( + kafka_topic: &KafkaTopic, + topic_name: &TopicName, + kafka_topics: &Store>, +) -> Option { + let key = |kafka_topic: &KafkaTopic| { + ( + kafka_topic.metadata.creation_timestamp.clone(), + kafka_topic.name_any(), + ) + }; + kafka_topics + .state() + .iter() + .filter_map(|other_kafka_topic| other_kafka_topic.0.as_ref().ok()) + .filter(|other_kafka_topic| is_same_topic(kafka_topic, other_kafka_topic, topic_name)) + .min_by_key(|other_kafka_topic| key(other_kafka_topic)) + .filter(|oldest_kafka_topic| key(oldest_kafka_topic) != key(kafka_topic)) + .map(ResourceExt::name_any) +} + +/// Whether another `KafkaTopic` that is not being deleted declares the same topic of the same +/// cluster, regardless of its age. +pub(crate) fn has_other_kafka_topic( + kafka_topic: &KafkaTopic, + topic_name: &TopicName, + kafka_topics: &Store>, +) -> bool { + kafka_topics + .state() + .iter() + .filter_map(|other_kafka_topic| other_kafka_topic.0.as_ref().ok()) + .any(|other_kafka_topic| { + other_kafka_topic.name_any() != kafka_topic.name_any() + && other_kafka_topic.metadata.deletion_timestamp.is_none() + && is_same_topic(kafka_topic, other_kafka_topic, topic_name) + }) +} + +fn is_same_topic( + kafka_topic: &KafkaTopic, + other_kafka_topic: &KafkaTopic, + topic_name: &TopicName, +) -> bool { + other_kafka_topic.spec.cluster_ref.name == kafka_topic.spec.cluster_ref.name + && other_kafka_topic + .topic_name() + .is_ok_and(|other_topic_name| &other_topic_name == topic_name) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use stackable_operator::{ + k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, + kube::runtime::watcher, + }; + + use super::*; + use crate::topic::test_support::kafka_topic; + + fn created_topic( + name: &str, + created_at: i64, + topic_name: Option<&str>, + ) -> DeserializeGuard { + let mut topic = kafka_topic(name, topic_name); + topic.metadata.creation_timestamp = Some(Time( + Timestamp::from_second(created_at).expect("the timestamp should be valid"), + )); + DeserializeGuard(Ok(topic)) + } + + fn store(topics: Vec>) -> Store> { + let (store, mut writer) = stackable_operator::kube::runtime::reflector::store(); + for topic in topics { + writer.apply_watcher_event(&watcher::Event::Apply(topic)); + } + store + } + + fn inner(topic: &DeserializeGuard) -> &KafkaTopic { + topic.0.as_ref().expect("the topic should be valid") + } + + fn topic_name(name: &str) -> TopicName { + TopicName::from_str(name).expect("the topic name should be valid") + } + + #[test] + fn the_oldest_kafka_topic_owns_the_topic() { + let older = created_topic("older", 10, Some("orders")); + let newer = created_topic("newer", 20, Some("orders")); + let other_topic = created_topic("other", 5, Some("payments")); + let topics = store(vec![older.clone(), newer.clone(), other_topic]); + + assert_eq!( + None, + conflicting_owner(inner(&older), &topic_name("orders"), &topics) + ); + assert_eq!( + Some("older".to_owned()), + conflicting_owner(inner(&newer), &topic_name("orders"), &topics) + ); + } + + #[test] + fn the_name_decides_between_kafka_topics_of_the_same_age() { + let first = created_topic("a", 10, Some("orders")); + let second = created_topic("b", 10, Some("orders")); + let topics = store(vec![second.clone(), first.clone()]); + + assert_eq!( + None, + conflicting_owner(inner(&first), &topic_name("orders"), &topics) + ); + assert_eq!( + Some("a".to_owned()), + conflicting_owner(inner(&second), &topic_name("orders"), &topics) + ); + } + + #[test] + fn compare_the_effective_topic_names() { + let explicit = created_topic("explicit", 10, Some("orders")); + let from_metadata_name = created_topic("orders", 20, None); + let topics = store(vec![explicit.clone(), from_metadata_name.clone()]); + + assert_eq!( + Some("explicit".to_owned()), + conflicting_owner(inner(&from_metadata_name), &topic_name("orders"), &topics) + ); + } + + #[test] + fn find_other_kafka_topics_regardless_of_their_age() { + let older = created_topic("older", 10, Some("orders")); + let newer = created_topic("newer", 20, Some("orders")); + let mut deleted = created_topic("deleted", 30, Some("payments")); + if let Ok(deleted) = &mut deleted.0 { + deleted.metadata.deletion_timestamp = deleted.metadata.creation_timestamp.clone(); + } + let payments = created_topic("payments", 40, None); + let topics = store(vec![older.clone(), newer, deleted, payments.clone()]); + + assert!(has_other_kafka_topic( + inner(&older), + &topic_name("orders"), + &topics + )); + assert!(!has_other_kafka_topic( + inner(&payments), + &topic_name("payments"), + &topics + )); + } + + #[rstest::rstest] + #[case::no_throttle(&[("retention.ms", "1000")], false)] + #[case::throttle(&[("follower.replication.throttled.replicas", "*")], true)] + fn reject_throttle_config_keys(#[case] config: &[(&str, &str)], #[case] invalid: bool) { + let mut topic = kafka_topic("orders", Some("orders")); + topic.spec.config_overrides = config + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + + assert_eq!( + invalid, + matches!( + validate(&topic, &store(vec![])), + Err(TopicSyncOutcome::InvalidSpec { .. }) + ) + ); + } + + #[test] + fn reject_a_too_long_metadata_name_without_topic_name() { + let topic = kafka_topic(&"a".repeat(250), None); + + assert!(matches!( + validate(&topic, &store(vec![])), + Err(TopicSyncOutcome::InvalidSpec { .. }) + )); + } +} From 624bebd61133324d2dc3980f0e28e015826bae4d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 13:54:24 +0100 Subject: [PATCH 11/33] add topic_controller to operator for cleaning up after agents --- .../templates/clusterrole-agent.yaml | 2 +- .../templates/clusterrole-operator.yaml | 18 + rust/agent-binary/src/topic/finalizer.rs | 10 +- rust/operator-binary/src/controller.rs | 13 +- rust/operator-binary/src/main.rs | 87 +++- rust/operator-binary/src/topic_controller.rs | 385 ++++++++++++++++++ rust/shared/src/crd/topic.rs | 4 + 7 files changed, 505 insertions(+), 14 deletions(-) create mode 100644 rust/operator-binary/src/topic_controller.rs diff --git a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml index 652fd210..0de7e9b8 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml @@ -8,7 +8,7 @@ metadata: labels: {{- include "kafka-operator.labels" . | nindent 4 }} rules: - # KafkaTopics of the agent's cluster: watched, annotated (ownership, managed config keys) and finalizers. + # KafkaTopics of the agent's cluster: watched, and their finalizer added and removed. - apiGroups: - {{ include "kafka-operator.name" . }}.stackable.tech resources: diff --git a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml index aafed503..f8f8193a 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml @@ -128,6 +128,24 @@ rules: - {{ include "kafka-operator.name" . }}clusters/status verbs: - patch + # KafkaTopics: watched to take over while no agent is available, and their finalizer removed + # once the KafkaCluster is gone. + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics + verbs: + - get + - list + - watch + - patch + # Status subresource: conditions set while the agent is unavailable. + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics/status + verbs: + - patch # Read authentication class configuration referenced in the KafkaCluster spec. - apiGroups: - authentication.stackable.tech diff --git a/rust/agent-binary/src/topic/finalizer.rs b/rust/agent-binary/src/topic/finalizer.rs index a26d8a47..95a65778 100644 --- a/rust/agent-binary/src/topic/finalizer.rs +++ b/rust/agent-binary/src/topic/finalizer.rs @@ -4,7 +4,7 @@ use std::path::Path; use serde_json::json; use snafu::{OptionExt, ResultExt}; -use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; +use stackable_kafka_shared::crd::topic::{KAFKA_TOPIC_FINALIZER, v1alpha1::KafkaTopic}; use stackable_operator::{ agent::deletion::v1alpha1::DeletionPolicy, kube::{ResourceExt, runtime::controller::Action}, @@ -22,8 +22,6 @@ use super::{ }; use crate::config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}; -const FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; - /// Handles a deletion and removes a finalizer that is no longer needed. Returns `Some` if the /// reconcile ends here. pub(crate) async fn reconcile_finalizer( @@ -69,7 +67,7 @@ fn has_finalizer(kafka_topic: &KafkaTopic) -> bool { kafka_topic .finalizers() .iter() - .any(|finalizer| finalizer == FINALIZER) + .any(|finalizer| finalizer == KAFKA_TOPIC_FINALIZER) } /// Deletes the topic in Kafka if the `deletionPolicy` says so and no other `KafkaTopic` declares @@ -117,11 +115,11 @@ async fn set_finalizer( let mut finalizers = kafka_topic .finalizers() .iter() - .filter(|finalizer| *finalizer != FINALIZER) + .filter(|finalizer| *finalizer != KAFKA_TOPIC_FINALIZER) .cloned() .collect::>(); if should_have_finalizer { - finalizers.push(FINALIZER.to_owned()); + finalizers.push(KAFKA_TOPIC_FINALIZER.to_owned()); } let patch = json!({ "metadata": { diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index b9618ede..0561030c 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -88,6 +88,14 @@ pub fn all_agents_selector() -> Selector { Selector::from_iter(BTreeMap::from(label::all_agents_selector(&PRODUCT_NAME))) } +/// Type-safe names for the resources of the agent of the given cluster. +pub fn agent_resource_names_of(cluster_name: &ClusterName) -> AgentResourceNames { + AgentResourceNames { + cluster_name: cluster_name.clone(), + product_name: PRODUCT_NAME.clone(), + } +} + #[derive(Snafu, Debug)] pub enum PodDescriptorsError { #[snafu(display( @@ -272,10 +280,7 @@ impl ValidatedCluster { /// Type-safe names for the resources of the agent. pub fn agent_resource_names(&self) -> AgentResourceNames { - AgentResourceNames { - cluster_name: self.name.clone(), - product_name: PRODUCT_NAME.clone(), - } + agent_resource_names_of(&self.name) } /// Type-safe names for the resources of a given role group. diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 07d45402..7df6ab26 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -1,13 +1,15 @@ // TODO: Look into how to properly resolve `clippy::large_enum_variant`. // This will need changes in our and upstream error types. -#![allow(clippy::result_large_err)] +#![allow(clippy::large_enum_variant, clippy::result_large_err)] use std::sync::Arc; use anyhow::anyhow; use clap::Parser; use futures::{FutureExt, StreamExt, TryFutureExt}; -use stackable_kafka_shared::crd::topic::{KafkaTopic, KafkaTopicVersion}; +use stackable_kafka_shared::crd::topic::{ + KafkaTopic, KafkaTopicVersion, v1alpha1 as topic_v1alpha1, +}; use stackable_operator::{ YamlSchema, cli::{Command, RunArguments}, @@ -40,11 +42,13 @@ use stackable_operator::{ use crate::{ controller::{KAFKA_FULL_CONTROLLER_NAME, all_agents_selector}, crd::{KAFKA_OPERATOR_NAME, KafkaCluster, KafkaClusterVersion, v1alpha1}, + topic_controller::{TOPIC_FULL_CONTROLLER_NAME, kafka_topics_of_cluster}, webhooks::conversion::create_webhook_server, }; mod controller; mod crd; +mod topic_controller; mod webhooks; mod built_info { @@ -133,6 +137,10 @@ async fn main() -> anyhow::Result<()> { "CRD {crd} established", crd = v1alpha1::KafkaCluster::crd_name() )); + let kafka_topic_crd_check = readiness_checks.register(format!( + "CRD {crd} established", + crd = topic_v1alpha1::KafkaTopic::crd_name() + )); let webhook_server = create_webhook_server( &operator_environment, @@ -234,13 +242,86 @@ async fn main() -> anyhow::Result<()> { ) .map(anyhow::Ok); + // Topic controller to clean up after deleted or unavailable agents + let topic_event_recorder = Arc::new(Recorder::new( + client.as_kube_client(), + Reporter { + controller: TOPIC_FULL_CONTROLLER_NAME.to_string(), + instance: None, + }, + )); + + let topic_controller = Controller::new( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + ); + let kafka_topic_store_for_clusters = topic_controller.store(); + let kafka_topic_store_for_agents = topic_controller.store(); + let topic_controller = topic_controller + .watches( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + move |kafka_cluster| { + kafka_topics_of_cluster( + &kafka_topic_store_for_clusters, + kafka_cluster.namespace(), + Some(&kafka_cluster.name_any()), + ) + }, + ) + .watches( + watch_namespace.get_api::>(&client), + watcher::Config::default().labels_from(&all_agents_selector()), + move |agent_deployment| { + kafka_topics_of_cluster( + &kafka_topic_store_for_agents, + agent_deployment.namespace(), + agent_deployment + .labels() + .get("app.kubernetes.io/instance") + .map(String::as_str), + ) + }, + ) + .graceful_shutdown_on(sigterm_watcher.handle()) + .run( + topic_controller::reconcile_topic, + topic_controller::error_policy, + Arc::new(topic_controller::Ctx { + client: client.clone(), + }), + ) + .for_each_concurrent(16, move |result| { + let topic_event_recorder = topic_event_recorder.clone(); + async move { + report_controller_reconciled( + &topic_event_recorder, + TOPIC_FULL_CONTROLLER_NAME, + &result, + ) + .await; + } + }) + .map(anyhow::Ok); + let delayed_kafka_controller = async { signal::crd_established(&client, v1alpha1::KafkaCluster::crd_name()).await?; kafka_cluster_crd_check.mark_passed(); kafka_controller.await }; - futures::try_join!(delayed_kafka_controller, eos_checker, webhook_server)?; + let delayed_topic_controller = async { + signal::crd_established(&client, topic_v1alpha1::KafkaTopic::crd_name()).await?; + kafka_topic_crd_check.mark_passed(); + topic_controller.await + }; + + futures::try_join!( + delayed_kafka_controller, + delayed_topic_controller, + eos_checker, + webhook_server + )?; } }; diff --git a/rust/operator-binary/src/topic_controller.rs b/rust/operator-binary/src/topic_controller.rs new file mode 100644 index 00000000..25414026 --- /dev/null +++ b/rust/operator-binary/src/topic_controller.rs @@ -0,0 +1,385 @@ +//! Takes over `KafkaTopic`s while no agent is available: reports it in their status and removes +//! their finalizer once the KafkaCluster is gone, so deleting the KafkaCluster or the namespace +//! does not block. + +use std::sync::Arc; + +use const_format::concatcp; +use serde_json::json; +use snafu::{ResultExt, Snafu}; +use stackable_kafka_shared::crd::topic::{KAFKA_TOPIC_FINALIZER, v1alpha1::KafkaTopic}; +use stackable_operator::{ + agent::condition::{ + ConditionStatus, ManagedCondition, ManagedConditionType, update_conditions, + }, + k8s_openapi::{ + api::apps::v1::Deployment, apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp, + }, + kube::{ + ResourceExt, + core::{DeserializeGuard, error_boundary}, + runtime::{ + controller::Action, + reflector::{ObjectRef, Store}, + }, + }, + logging::controller::ReconcilerError, + shared::time::Duration, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::{ + controller::agent_resource_names_of, + crd::{KAFKA_OPERATOR_NAME, v1alpha1::KafkaCluster}, +}; + +pub const TOPIC_CONTROLLER_NAME: &str = "kafkatopic"; +pub const TOPIC_FULL_CONTROLLER_NAME: &str = + concatcp!(TOPIC_CONTROLLER_NAME, '.', KAFKA_OPERATOR_NAME); + +const AGENT_UNAVAILABLE_REASON: &str = "AgentUnavailable"; + +#[derive(Debug, Snafu, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("the KafkaTopic is invalid"))] + InvalidKafkaTopic { + source: error_boundary::InvalidObject, + }, + + #[snafu(display("failed to get the KafkaCluster"))] + GetKafkaCluster { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to get the agent Deployment"))] + GetAgentDeployment { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to update the status of the KafkaTopic"))] + UpdateStatus { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to remove the finalizer of the KafkaTopic"))] + RemoveFinalizer { + source: stackable_operator::client::Error, + }, +} + +type Result = std::result::Result; + +impl ReconcilerError for Error { + fn category(&self) -> &'static str { + ErrorDiscriminants::from(self).into() + } +} + +pub struct Ctx { + pub client: stackable_operator::client::Client, +} + +/// The KafkaCluster of a `KafkaTopic`, as far as its agent is concerned. +#[derive(Debug, Eq, PartialEq)] +enum ClusterState { + /// Does not exist or has a `deletionTimestamp` + MissingOrDeleting, + ManagementAccessDisabled, + ManagementAccessEnabled, +} + +#[derive(Debug, Eq, PartialEq)] +enum OperatorAction { + LeaveToAgent, + RemoveFinalizer, + SetAgentUnavailableConditions { message: &'static str }, +} + +pub async fn reconcile_topic( + kafka_topic: Arc>, + ctx: Arc, +) -> Result { + let kafka_topic = kafka_topic + .0 + .as_ref() + .map_err(error_boundary::InvalidObject::clone) + .context(InvalidKafkaTopicSnafu)?; + let namespace = kafka_topic.namespace().unwrap_or_default(); + let cluster_name = &kafka_topic.spec.cluster_ref.name; + + let kafka_cluster = ctx + .client + .get_opt::>(cluster_name.as_ref(), &namespace) + .await + .context(GetKafkaClusterSnafu)?; + let agent_deployment_name = agent_resource_names_of(cluster_name).deployment_name(); + let agent_deployment = ctx + .client + .get_opt::(agent_deployment_name.as_ref(), &namespace) + .await + .context(GetAgentDeploymentSnafu)?; + + match operator_action_for( + kafka_topic, + &cluster_state_from(kafka_cluster.as_ref()), + agent_deployment + .as_ref() + .is_some_and(is_agent_deployment_available), + ) { + OperatorAction::LeaveToAgent => {} + OperatorAction::RemoveFinalizer => remove_finalizer(kafka_topic, &ctx).await?, + OperatorAction::SetAgentUnavailableConditions { message } => { + set_agent_unavailable_conditions(kafka_topic, &ctx, message).await? + } + } + Ok(Action::await_change()) +} + +pub fn error_policy( + _kafka_topic: Arc>, + error: &Error, + _ctx: Arc, +) -> Action { + match error { + Error::InvalidKafkaTopic { .. } => Action::await_change(), + _ => Action::requeue(*Duration::from_secs(30)), + } +} + +/// The `KafkaTopic`s of the KafkaCluster `cluster_name`, to reconcile them when the KafkaCluster +/// or its agent changes. +pub fn kafka_topics_of_cluster( + kafka_topics: &Store>, + namespace: Option, + cluster_name: Option<&str>, +) -> Vec>> { + kafka_topics + .state() + .into_iter() + .filter(|kafka_topic| { + kafka_topic.0.as_ref().is_ok_and(|kafka_topic| { + kafka_topic.namespace() == namespace + && Some(kafka_topic.spec.cluster_ref.name.as_ref()) == cluster_name + }) + }) + .map(|kafka_topic| ObjectRef::from_obj(&*kafka_topic)) + .collect() +} + +fn cluster_state_from(kafka_cluster: Option<&DeserializeGuard>) -> ClusterState { + match kafka_cluster.map(|kafka_cluster| &kafka_cluster.0) { + None => ClusterState::MissingOrDeleting, + Some(Ok(kafka_cluster)) if kafka_cluster.metadata.deletion_timestamp.is_some() => { + ClusterState::MissingOrDeleting + } + Some(Ok(kafka_cluster)) + if !kafka_cluster + .spec + .cluster_config + .management_access + .as_ref() + .is_some_and(|management_access| management_access.enabled) => + { + ClusterState::ManagementAccessDisabled + } + // An invalid KafkaCluster keeps its agent, so the state of the agent decides. + Some(_) => ClusterState::ManagementAccessEnabled, + } +} + +fn operator_action_for( + kafka_topic: &KafkaTopic, + cluster_state: &ClusterState, + agent_available: bool, +) -> OperatorAction { + match cluster_state { + ClusterState::MissingOrDeleting if kafka_topic.metadata.deletion_timestamp.is_some() => { + if kafka_topic + .finalizers() + .iter() + .any(|finalizer| finalizer == KAFKA_TOPIC_FINALIZER) + { + OperatorAction::RemoveFinalizer + } else { + OperatorAction::LeaveToAgent + } + } + ClusterState::MissingOrDeleting => OperatorAction::SetAgentUnavailableConditions { + message: "the KafkaCluster does not exist", + }, + ClusterState::ManagementAccessDisabled => OperatorAction::SetAgentUnavailableConditions { + message: "the KafkaCluster has no agent, because clusterConfig.managementAccess is \ + not enabled", + }, + ClusterState::ManagementAccessEnabled if agent_available => OperatorAction::LeaveToAgent, + ClusterState::ManagementAccessEnabled => OperatorAction::SetAgentUnavailableConditions { + message: "the agent of the KafkaCluster is not available", + }, + } +} + +fn is_agent_deployment_available(agent_deployment: &Deployment) -> bool { + agent_deployment + .status + .as_ref() + .and_then(|status| status.conditions.as_ref()) + .is_some_and(|conditions| { + conditions + .iter() + .any(|condition| condition.type_ == "Available" && condition.status == "True") + }) +} + +/// Only the conditions are applied, so the other fields stay owned by the agent and keep the +/// last state it observed. +async fn set_agent_unavailable_conditions( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + message: &str, +) -> Result<()> { + let current_conditions = kafka_topic + .status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or_default(); + let new_conditions = update_conditions( + current_conditions, + vec![ + agent_unavailable_condition( + ManagedConditionType::Ready, + ConditionStatus::Unknown, + message, + ), + agent_unavailable_condition( + ManagedConditionType::Synced, + ConditionStatus::False, + message, + ), + ], + &Time(Timestamp::now()), + kafka_topic.metadata.generation, + ); + + if new_conditions != current_conditions { + ctx.client + .apply_patch_status( + TOPIC_CONTROLLER_NAME, + kafka_topic, + &json!({ "conditions": new_conditions }), + ) + .await + .context(UpdateStatusSnafu)?; + } + Ok(()) +} + +fn agent_unavailable_condition( + type_: ManagedConditionType, + status: ConditionStatus, + message: &str, +) -> ManagedCondition { + ManagedCondition { + type_, + status, + reason: AGENT_UNAVAILABLE_REASON.to_owned(), + message: message.to_owned(), + } +} + +/// Fails if the `KafkaTopic` was changed since it was read, so concurrent changes to the +/// finalizers, e.g. by an agent that is still shutting down, are not lost. +async fn remove_finalizer(kafka_topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + let finalizers = kafka_topic + .finalizers() + .iter() + .filter(|finalizer| *finalizer != KAFKA_TOPIC_FINALIZER) + .cloned() + .collect::>(); + let patch = json!({ + "metadata": { + "resourceVersion": kafka_topic.metadata.resource_version, + "finalizers": finalizers, + } + }); + ctx.client + .merge_patch(kafka_topic, patch) + .await + .context(RemoveFinalizerSnafu)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use stackable_operator::kube::api::ObjectMeta; + + use super::*; + + fn kafka_topic(deleting: bool, with_finalizer: bool) -> KafkaTopic { + let kafka_topic: KafkaTopic = serde_json::from_value(json!({ + "apiVersion": "kafka.stackable.tech/v1alpha1", + "kind": "KafkaTopic", + "metadata": { "name": "orders", "namespace": "default" }, + "spec": { "clusterRef": { "name": "simple-kafka" } }, + })) + .expect("the KafkaTopic should be valid"); + KafkaTopic { + metadata: ObjectMeta { + deletion_timestamp: deleting.then_some(Time(Timestamp::UNIX_EPOCH)), + finalizers: with_finalizer.then(|| vec![KAFKA_TOPIC_FINALIZER.to_owned()]), + ..kafka_topic.metadata + }, + ..kafka_topic + } + } + + #[rstest] + #[case::cluster_missing(ClusterState::MissingOrDeleting, true, true)] + #[case::cluster_missing_without_agent(ClusterState::MissingOrDeleting, false, true)] + #[case::management_access_disabled(ClusterState::ManagementAccessDisabled, false, false)] + #[case::agent_available(ClusterState::ManagementAccessEnabled, true, false)] + #[case::agent_unavailable(ClusterState::ManagementAccessEnabled, false, false)] + fn remove_the_finalizer_only_if_the_cluster_is_missing_or_deleting( + #[case] cluster_state: ClusterState, + #[case] agent_available: bool, + #[case] removes_finalizer: bool, + ) { + assert_eq!( + removes_finalizer, + operator_action_for(&kafka_topic(true, true), &cluster_state, agent_available) + == OperatorAction::RemoveFinalizer + ); + } + + #[rstest] + #[case::cluster_missing(ClusterState::MissingOrDeleting, true, true)] + #[case::management_access_disabled(ClusterState::ManagementAccessDisabled, true, true)] + #[case::agent_available(ClusterState::ManagementAccessEnabled, true, false)] + #[case::agent_unavailable(ClusterState::ManagementAccessEnabled, false, true)] + fn set_agent_unavailable_conditions_without_an_available_agent( + #[case] cluster_state: ClusterState, + #[case] agent_available: bool, + #[case] sets_conditions: bool, + ) { + assert_eq!( + sets_conditions, + matches!( + operator_action_for(&kafka_topic(false, false), &cluster_state, agent_available), + OperatorAction::SetAgentUnavailableConditions { .. } + ) + ); + } + + #[test] + fn leave_a_deleting_kafka_topic_without_finalizer_alone() { + assert_eq!( + OperatorAction::LeaveToAgent, + operator_action_for( + &kafka_topic(true, false), + &ClusterState::MissingOrDeleting, + false + ) + ); + } +} diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 01600daa..3818f032 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -11,6 +11,10 @@ use stackable_operator::{ versioned::versioned, }; +/// Set by the agent on a `KafkaTopic` that deletes its topic. Removed by the agent after the +/// deletion, or by the operator once the KafkaCluster is gone, so no agent can remove it anymore. +pub const KAFKA_TOPIC_FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; + attributed_string_type! { TopicName, "The name of a Kafka topic, not `.` or `..`", From ac90b9f9a771ab688aee5e2518d8c6a9166ff530 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 15:14:33 +0100 Subject: [PATCH 12/33] read bootstrap servers from KAFKA_BOOTSTRAP_SERVERS in the agent and create client only once --- rust/agent-binary/src/config.rs | 96 +------------------ rust/agent-binary/src/main.rs | 18 +++- rust/agent-binary/src/topic/condition.rs | 12 --- rust/agent-binary/src/topic/controller.rs | 24 ++--- rust/agent-binary/src/topic/finalizer.rs | 19 +--- rust/agent-binary/src/topic/sync.rs | 62 ++---------- .../controller/build/properties/listener.rs | 2 - rust/operator-binary/src/topic_controller.rs | 3 - 8 files changed, 36 insertions(+), 200 deletions(-) diff --git a/rust/agent-binary/src/config.rs b/rust/agent-binary/src/config.rs index 0f3319e9..9c2dcc47 100644 --- a/rust/agent-binary/src/config.rs +++ b/rust/agent-binary/src/config.rs @@ -1,38 +1,14 @@ //! The agent's view of the files and settings the operator provides. -use std::{ - fs, io, - path::{Path, PathBuf}, -}; - use const_format::concatcp; -use snafu::{ResultExt, Snafu}; -use stackable_kafka_admin::{ - bootstrap::{self, BrokerAddress, parse_bootstrap_servers}, - connection::Security, -}; -use stackable_kafka_shared::agent::{ClientSecurity, DISCOVERY_DIR, KAFKA_TLS_DIR}; +use stackable_kafka_admin::connection::Security; +use stackable_kafka_shared::agent::{ClientSecurity, KAFKA_TLS_DIR}; use stackable_operator::crd::management_access::tls::MOUNT_PATH; -/// The key `KAFKA` of the discovery ConfigMap, holding `host:port,host:port` -pub const BOOTSTRAP_SERVERS_FILE: &str = concatcp!(DISCOVERY_DIR, "/KAFKA"); - const CA_FILE: &str = concatcp!(KAFKA_TLS_DIR, "/ca.crt"); const CERTIFICATE_FILE: &str = concatcp!(MOUNT_PATH, "/tls.crt"); const KEY_FILE: &str = concatcp!(MOUNT_PATH, "/tls.key"); -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("failed to read the bootstrap servers from {path:?}"))] - ReadBootstrapServers { source: io::Error, path: PathBuf }, - - #[snafu(display("the bootstrap servers in {path:?} are invalid"))] - ParseBootstrapServers { - source: bootstrap::Error, - path: PathBuf, - }, -} - pub fn security(client_security: ClientSecurity) -> Security { match client_security { ClientSecurity::Plaintext => Security::Plaintext, @@ -45,17 +21,6 @@ pub fn security(client_security: ClientSecurity) -> Security { } } -/// Returns `None` while the operator has written no bootstrap servers to the discovery ConfigMap yet. -pub fn read_bootstrap_servers(path: &Path) -> Result>, Error> { - let value = fs::read_to_string(path).context(ReadBootstrapServersSnafu { path })?; - if value.trim().is_empty() { - return Ok(None); - } - parse_bootstrap_servers(&value) - .map(Some) - .context(ParseBootstrapServersSnafu { path }) -} - #[cfg(test)] mod tests { use rstest::rstest; @@ -81,61 +46,4 @@ mod tests { fn map_client_security(#[case] client_security: ClientSecurity, #[case] expected: Security) { assert_eq!(expected, security(client_security)); } - - #[rstest] - #[case::empty("", None)] - #[case::blank(" \n", None)] - #[case::servers( - "kafka-0:9093,kafka-1:9093", - Some(vec![ - BrokerAddress { host: "kafka-0".to_owned(), port: 9093 }, - BrokerAddress { host: "kafka-1".to_owned(), port: 9093 }, - ]) - )] - fn read_written_bootstrap_servers( - #[case] content: &str, - #[case] expected: Option>, - ) { - let path = temp_file(content); - - let bootstrap_servers = read_bootstrap_servers(&path); - fs::remove_file(&path).expect("the file should be removable"); - - assert_eq!( - expected, - bootstrap_servers.expect("the bootstrap servers should be readable") - ); - } - - #[test] - fn reject_missing_bootstrap_servers_file() { - let path = std::env::temp_dir().join("missing-bootstrap-servers"); - - assert!(matches!( - read_bootstrap_servers(&path).expect_err("a missing file should be an error"), - Error::ReadBootstrapServers { .. } - )); - } - - #[test] - fn reject_invalid_bootstrap_servers() { - let path = temp_file("kafka"); - - let error = read_bootstrap_servers(&path); - fs::remove_file(&path).expect("the file should be removable"); - - assert!(matches!( - error.expect_err("the bootstrap servers should be invalid"), - Error::ParseBootstrapServers { .. } - )); - } - - /// Writes the content to a file unique to this test thread. - fn temp_file(content: &str) -> PathBuf { - let thread = std::thread::current(); - let name = thread.name().unwrap_or("test").replace("::", "-"); - let path = std::env::temp_dir().join(format!("bootstrap-{}-{name}", std::process::id())); - fs::write(&path, content).expect("the file should be writable"); - path - } } diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 5b61c878..2c950f16 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -6,8 +6,12 @@ use std::sync::Arc; use clap::Parser; use futures::StreamExt; +use stackable_kafka_admin::{bootstrap::parse_bootstrap_servers, client::KafkaAdminClient}; use stackable_kafka_shared::{ - agent::{ClientSecurity, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE}, + agent::{ + ClientSecurity, ENV_BOOTSTRAP_SERVERS, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, + ENV_CLUSTER_NAMESPACE, + }, crd::topic::v1alpha1::KafkaTopic, }; use stackable_operator::{ @@ -34,6 +38,8 @@ mod topic; pub const AGENT_NAME: &str = "kafka-agent.stackable.tech"; +const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; + mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } @@ -59,6 +65,10 @@ struct KafkaAgentRun { #[arg(long, env = AsRef::::as_ref(&*ENV_CLIENT_SECURITY))] client_security: ClientSecurity, + /// The brokers to connect to, as `host:port,host:port`. + #[arg(long, env = AsRef::::as_ref(&*ENV_BOOTSTRAP_SERVERS))] + bootstrap_servers: String, + #[clap(flatten)] common: CommonOptions, } @@ -71,6 +81,7 @@ async fn main() -> anyhow::Result<()> { cluster_name, cluster_namespace, client_security, + bootstrap_servers, common: CommonOptions { telemetry, @@ -90,12 +101,15 @@ async fn main() -> anyhow::Result<()> { ); let security = config::security(client_security); + let bootstrap_servers = parse_bootstrap_servers(&bootstrap_servers)?; tracing::info!( %cluster_name, %cluster_namespace, ?security, + ?bootstrap_servers, "Configured the agent" ); + let admin_client = KafkaAdminClient::new(bootstrap_servers, security, KAFKA_CLIENT_ID); // Watches for the SIGTERM signal and gracefully shuts down the controller. let sigterm_watcher = SignalWatcher::sigterm()?; @@ -124,7 +138,7 @@ async fn main() -> anyhow::Result<()> { Arc::new(controller::Ctx::new( client, cluster_name, - security, + admin_client, kafka_topic_store, )), ) diff --git a/rust/agent-binary/src/topic/condition.rs b/rust/agent-binary/src/topic/condition.rs index d1c6fb8b..46ed4f80 100644 --- a/rust/agent-binary/src/topic/condition.rs +++ b/rust/agent-binary/src/topic/condition.rs @@ -7,10 +7,6 @@ use stackable_operator::agent::condition::{ use super::plan::{TopicState, UnsupportedChange}; -/// The operator writes the bootstrap servers to the discovery ConfigMap only once the -/// listener-operator has assigned addresses to the bootstrap Listeners. -const CLUSTER_NOT_READY_MESSAGE: &str = "the KafkaCluster has no bootstrap servers yet"; - #[derive(Debug, Eq, PartialEq)] pub enum TopicSyncOutcome { /// The topic matches the spec @@ -22,7 +18,6 @@ pub enum TopicSyncOutcome { changes: Vec, }, ReconciliationPaused, - ClusterNotReady, InvalidSpec { message: String, }, @@ -87,9 +82,6 @@ fn ready_from( TopicSyncOutcome::Conflict { .. } => { ready(False, "Conflict", "another KafkaTopic manages the topic") } - TopicSyncOutcome::ClusterNotReady => { - ready(Unknown, "ClusterNotReady", CLUSTER_NOT_READY_MESSAGE) - } TopicSyncOutcome::InvalidSpec { message } => ready(Unknown, "InvalidSpec", message), TopicSyncOutcome::KafkaRejected { message } => ready(Unknown, "KafkaRejected", message), TopicSyncOutcome::ConnectionFailed { message } => { @@ -129,9 +121,6 @@ fn synced_from( "the reconciliation is paused, only the status is updated", ), TopicSyncOutcome::Conflict { message } => synced(False, "Conflict", message), - TopicSyncOutcome::ClusterNotReady => { - synced(False, "ClusterNotReady", CLUSTER_NOT_READY_MESSAGE) - } TopicSyncOutcome::InvalidSpec { message } => synced(False, "InvalidSpec", message), TopicSyncOutcome::KafkaRejected { message } => synced(False, "KafkaRejected", message), TopicSyncOutcome::ConnectionFailed { message } => { @@ -338,7 +327,6 @@ mod tests { } #[rstest::rstest] - #[case::cluster_not_ready(TopicSyncOutcome::ClusterNotReady, "ClusterNotReady")] #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: "invalid".to_owned() }, "InvalidSpec")] #[case::kafka_rejected(TopicSyncOutcome::KafkaRejected { message: "denied".to_owned() }, "KafkaRejected")] #[case::connection_failed(TopicSyncOutcome::ConnectionFailed { message: "timed out".to_owned() }, "ConnectionFailed")] diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs index 105ac03c..8d50d046 100644 --- a/rust/agent-binary/src/topic/controller.rs +++ b/rust/agent-binary/src/topic/controller.rs @@ -4,7 +4,7 @@ use std::{sync::Arc, time::Duration}; use const_format::concatcp; use snafu::{ResultExt, Snafu}; -use stackable_kafka_admin::{client, connection::Security}; +use stackable_kafka_admin::client::{self, KafkaAdminClient}; use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; use stackable_operator::{ kube::{ @@ -21,7 +21,7 @@ use tracing::info; use super::{ condition::TopicSyncOutcome, finalizer::{add_finalizer, reconcile_finalizer}, - sync::{CachedClient, sync}, + sync::sync, update_status::update_status, }; use crate::AGENT_NAME; @@ -30,7 +30,6 @@ pub const TOPIC_CONTROLLER_NAME: &str = "kafkatopic"; pub const TOPIC_FULL_CONTROLLER_NAME: &str = concatcp!(TOPIC_CONTROLLER_NAME, ".", AGENT_NAME); const RESYNC_INTERVAL: Duration = Duration::from_secs(5 * 60); -const RETRY_INTERVAL: Duration = Duration::from_secs(10); const ERROR_RETRY_INTERVAL: Duration = Duration::from_secs(30); #[derive(Debug, Snafu, EnumDiscriminants)] @@ -42,12 +41,6 @@ pub enum Error { source: error_boundary::InvalidObject, }, - #[snafu(display("failed to read the bootstrap servers"))] - ReadBootstrapServers { source: crate::config::Error }, - - #[snafu(display("the KafkaCluster has no bootstrap servers yet"))] - NoBootstrapServers, - #[snafu(display("failed to talk to Kafka"))] Kafka { source: client::Error }, @@ -73,10 +66,10 @@ impl ReconcilerError for Error { pub struct Ctx { pub(crate) client: stackable_operator::client::Client, pub(crate) cluster_name: ClusterName, - pub(crate) security: Security, pub(crate) kafka_topics: Store>, - /// One client per agent, rebuilt when the bootstrap servers change - pub(crate) client_cache: Mutex>, + /// Reads the TLS files on every new connection and reconnects after a failed request, so + /// rotated certificates are picked up without restarting the agent. + pub(crate) admin_client: Mutex, } impl Ctx { @@ -85,15 +78,14 @@ impl Ctx { pub fn new( client: stackable_operator::client::Client, cluster_name: ClusterName, - security: Security, + admin_client: KafkaAdminClient, kafka_topics: Store>, ) -> Self { Self { client, cluster_name, - security, kafka_topics, - client_cache: Mutex::new(None), + admin_client: Mutex::new(admin_client), } } } @@ -150,7 +142,6 @@ pub async fn reconcile_topic( fn requeue(sync_outcome: &TopicSyncOutcome) -> Action { match sync_outcome { TopicSyncOutcome::InvalidSpec { .. } => Action::await_change(), - TopicSyncOutcome::ClusterNotReady => Action::requeue(RETRY_INTERVAL), _ => Action::requeue(RESYNC_INTERVAL), } } @@ -172,7 +163,6 @@ mod tests { #[rstest::rstest] #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: String::new() }, Action::await_change())] - #[case::cluster_not_ready(TopicSyncOutcome::ClusterNotReady, Action::requeue(RETRY_INTERVAL))] #[case::synced(TopicSyncOutcome::Synced { reassigning_partition_count: 0 }, Action::requeue(RESYNC_INTERVAL))] #[case::paused( TopicSyncOutcome::ReconciliationPaused, diff --git a/rust/agent-binary/src/topic/finalizer.rs b/rust/agent-binary/src/topic/finalizer.rs index 95a65778..1eb9fc39 100644 --- a/rust/agent-binary/src/topic/finalizer.rs +++ b/rust/agent-binary/src/topic/finalizer.rs @@ -1,9 +1,7 @@ //! The finalizer of a `KafkaTopic` and the cleanup in Kafka when it is deleted. -use std::path::Path; - use serde_json::json; -use snafu::{OptionExt, ResultExt}; +use snafu::ResultExt; use stackable_kafka_shared::crd::topic::{KAFKA_TOPIC_FINALIZER, v1alpha1::KafkaTopic}; use stackable_operator::{ agent::deletion::v1alpha1::DeletionPolicy, @@ -12,15 +10,10 @@ use stackable_operator::{ use tracing::info; use super::{ - controller::{ - Ctx, KafkaSnafu, NoBootstrapServersSnafu, ReadBootstrapServersSnafu, Result, - UpdateFinalizersSnafu, - }, + controller::{Ctx, KafkaSnafu, Result, UpdateFinalizersSnafu}, plan::TopicState, - sync::client_for, validate::has_other_kafka_topic, }; -use crate::config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}; /// Handles a deletion and removes a finalizer that is no longer needed. Returns `Some` if the /// reconcile ends here. @@ -87,11 +80,9 @@ async fn cleanup(kafka_topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { return Ok(()); } - let bootstrap_servers = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) - .context(ReadBootstrapServersSnafu)? - .context(NoBootstrapServersSnafu)?; - let mut client_cache = ctx.client_cache.lock().await; - client_for(&mut client_cache, bootstrap_servers, &ctx.security) + ctx.admin_client + .lock() + .await .delete_topic(topic_name.as_ref()) .await .context(KafkaSnafu)?; diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs index 7608efe9..4acd0d27 100644 --- a/rust/agent-binary/src/topic/sync.rs +++ b/rust/agent-binary/src/topic/sync.rs @@ -1,34 +1,19 @@ //! Brings the topic in Kafka to the state of its `KafkaTopic` spec. -use std::{ - collections::{BTreeMap, BTreeSet}, - path::Path, -}; +use std::collections::{BTreeMap, BTreeSet}; use snafu::ResultExt; -use stackable_kafka_admin::{ - bootstrap::BrokerAddress, - client::{KafkaAdminClient, TopicConfig}, - connection::Security, -}; +use stackable_kafka_admin::client::{KafkaAdminClient, TopicConfig}; use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopic}; use stackable_operator::kube::ResourceExt; use tracing::info; use super::{ condition::TopicSyncOutcome, - controller::{Ctx, KafkaSnafu, ReadBootstrapServersSnafu, Result}, + controller::{Ctx, KafkaSnafu, Result}, plan::{ObservedTopic, TopicPlan, TopicState, plan}, validate::validate, }; -use crate::config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}; - -const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; - -pub(crate) struct CachedClient { - bootstrap_servers: Vec, - admin_client: KafkaAdminClient, -} /// Sets `topic_state` to the topic in Kafka, if it exists and is managed by this `KafkaTopic`. /// It is kept up to date with each applied change, also if a later step fails. @@ -44,14 +29,8 @@ pub(crate) async fn sync( }; // observe (Kafka) - let Some(bootstrap_servers) = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) - .context(ReadBootstrapServersSnafu)? - else { - return Ok(TopicSyncOutcome::ClusterNotReady); - }; - let mut client_cache = ctx.client_cache.lock().await; - let admin_client = client_for(&mut client_cache, bootstrap_servers, &ctx.security); - let observed = observe(admin_client, &topic_name).await?; + let mut admin_client = ctx.admin_client.lock().await; + let observed = observe(&mut admin_client, &topic_name).await?; // plan (no client required) let topic_plan = plan(&kafka_topic.spec, observed.as_ref()); @@ -66,7 +45,7 @@ pub(crate) async fn sync( // execute (Kafka) execute( kafka_topic, - admin_client, + &mut admin_client, topic_name, &topic_plan, topic_state, @@ -249,32 +228,3 @@ async fn replace_configs( .await .context(KafkaSnafu) } - -/// Returns the cached client, rebuilt if the bootstrap servers changed. -/// -/// The client reads the TLS files on every new connection and reconnects after a failed request, -/// so rotated certificates are picked up without restarting the agent. -pub(crate) fn client_for<'a>( - client_cache: &'a mut Option, - bootstrap_servers: Vec, - security: &Security, -) -> &'a mut KafkaAdminClient { - if client_cache - .as_ref() - .is_some_and(|cached| cached.bootstrap_servers != bootstrap_servers) - { - *client_cache = None; - } - let cached = client_cache.get_or_insert_with(|| { - info!(?bootstrap_servers, "Connecting to new bootstrap servers"); - CachedClient { - bootstrap_servers: bootstrap_servers.clone(), - admin_client: KafkaAdminClient::new( - bootstrap_servers, - security.clone(), - KAFKA_CLIENT_ID, - ), - } - }); - &mut cached.admin_client -} diff --git a/rust/operator-binary/src/controller/build/properties/listener.rs b/rust/operator-binary/src/controller/build/properties/listener.rs index 2cb53376..e7a460b7 100644 --- a/rust/operator-binary/src/controller/build/properties/listener.rs +++ b/rust/operator-binary/src/controller/build/properties/listener.rs @@ -157,8 +157,6 @@ pub(crate) fn pod_fqdn( format!("${{env:POD_NAME}}.{sts_service_name}.{namespace}.svc.{cluster_domain}") } -/// `..svc.:`, which resolves to the Pods of a headless -/// Service. pub(crate) fn headless_service_address( headless_service_name: &ServiceName, namespace: &NamespaceName, diff --git a/rust/operator-binary/src/topic_controller.rs b/rust/operator-binary/src/topic_controller.rs index 25414026..e724e2a9 100644 --- a/rust/operator-binary/src/topic_controller.rs +++ b/rust/operator-binary/src/topic_controller.rs @@ -80,7 +80,6 @@ pub struct Ctx { pub client: stackable_operator::client::Client, } -/// The KafkaCluster of a `KafkaTopic`, as far as its agent is concerned. #[derive(Debug, Eq, PartialEq)] enum ClusterState { /// Does not exist or has a `deletionTimestamp` @@ -147,8 +146,6 @@ pub fn error_policy( } } -/// The `KafkaTopic`s of the KafkaCluster `cluster_name`, to reconcile them when the KafkaCluster -/// or its agent changes. pub fn kafka_topics_of_cluster( kafka_topics: &Store>, namespace: Option, From 3fb16debcfdc67d52fefca1984614abf208ed662 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 15:26:50 +0100 Subject: [PATCH 13/33] skip planning while the KafkaTopic reconciliation is paused --- rust/agent-binary/src/topic/plan.rs | 9 ++++----- rust/agent-binary/src/topic/sync.rs | 10 ++++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs index 81e4ce02..a52b21ec 100644 --- a/rust/agent-binary/src/topic/plan.rs +++ b/rust/agent-binary/src/topic/plan.rs @@ -32,14 +32,13 @@ pub struct TopicState { } impl TopicState { - pub fn observed(name: TopicName, observed: ObservedTopic) -> Self { - let replication_factor = replication_factor(&observed); + pub fn observed(name: TopicName, observed: &ObservedTopic) -> Self { Self { name, - id: observed.topic.id, + id: observed.topic.id.clone(), partition_count: observed.topic.partitions.len() as i32, - replication_factor, - effective_configs: observed.effective_configs, + replication_factor: replication_factor(observed), + effective_configs: observed.effective_configs.clone(), } } diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs index 4acd0d27..fd34efb0 100644 --- a/rust/agent-binary/src/topic/sync.rs +++ b/rust/agent-binary/src/topic/sync.rs @@ -31,16 +31,18 @@ pub(crate) async fn sync( // observe (Kafka) let mut admin_client = ctx.admin_client.lock().await; let observed = observe(&mut admin_client, &topic_name).await?; + *topic_state = observed + .as_ref() + .map(|observed| TopicState::observed(topic_name.clone(), observed)); + if kafka_topic.spec.topic_operation.reconciliation_paused { + return Ok(TopicSyncOutcome::ReconciliationPaused); + } // plan (no client required) let topic_plan = plan(&kafka_topic.spec, observed.as_ref()); let reassigning_partition_count = observed .as_ref() .map_or(0, |observed| observed.reassignments.len()); - *topic_state = observed.map(|observed| TopicState::observed(topic_name.clone(), observed)); - if kafka_topic.spec.topic_operation.reconciliation_paused { - return Ok(TopicSyncOutcome::ReconciliationPaused); - } // execute (Kafka) execute( From b3e247d1badb2aa88db5ee3c4a2a7bc5900ca6af Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 15:30:28 +0100 Subject: [PATCH 14/33] watch only the KafkaTopics of the agent's cluster via a field selector --- rust/agent-binary/src/main.rs | 4 ++-- rust/agent-binary/src/topic/controller.rs | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 2c950f16..48efbb2d 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -127,7 +127,8 @@ async fn main() -> anyhow::Result<()> { let topic_controller = Controller::new( client.get_api::>(cluster_namespace.as_ref()), - watcher::Config::default(), + // Works because the CRD declares the field as a selectableField. + watcher::Config::default().fields(&format!("spec.clusterRef.name={cluster_name}")), ); let kafka_topic_store = topic_controller.store(); topic_controller @@ -137,7 +138,6 @@ async fn main() -> anyhow::Result<()> { controller::error_policy, Arc::new(controller::Ctx::new( client, - cluster_name, admin_client, kafka_topic_store, )), diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs index 8d50d046..cdc9a496 100644 --- a/rust/agent-binary/src/topic/controller.rs +++ b/rust/agent-binary/src/topic/controller.rs @@ -12,7 +12,6 @@ use stackable_operator::{ runtime::{controller::Action, reflector::Store}, }, logging::controller::ReconcilerError, - v2::types::operator::ClusterName, }; use strum::{EnumDiscriminants, IntoStaticStr}; use tokio::sync::Mutex; @@ -65,7 +64,6 @@ impl ReconcilerError for Error { pub struct Ctx { pub(crate) client: stackable_operator::client::Client, - pub(crate) cluster_name: ClusterName, pub(crate) kafka_topics: Store>, /// Reads the TLS files on every new connection and reconnects after a failed request, so /// rotated certificates are picked up without restarting the agent. @@ -77,13 +75,11 @@ impl Ctx { /// `KafkaTopic`s. pub fn new( client: stackable_operator::client::Client, - cluster_name: ClusterName, admin_client: KafkaAdminClient, kafka_topics: Store>, ) -> Self { Self { client, - cluster_name, kafka_topics, admin_client: Mutex::new(admin_client), } @@ -102,9 +98,6 @@ pub async fn reconcile_topic( .as_ref() .map_err(error_boundary::InvalidObject::clone) .context(InvalidKafkaTopicSnafu)?; - if kafka_topic.spec.cluster_ref.name != ctx.cluster_name { - return Ok(Action::await_change()); - } // finalizer (Kubernetes) // Left untouched while paused, like operators that only update the status then. A deletion From 6b9300431e29f69baed8fc16175c5f30af53206b Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 15:52:42 +0100 Subject: [PATCH 15/33] reject topic names starting with __ (internal topics) --- extra/crds.yaml | 3 +++ rust/shared/src/crd/topic.rs | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/extra/crds.yaml b/extra/crds.yaml index d8d32e1c..d7b52284 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2325,6 +2325,9 @@ spec: nullable: true pattern: ^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$ type: string + x-kubernetes-validations: + - message: topics starting with __ are internal to Kafka + rule: '!self.startsWith(''__'')' topicOperation: default: reconciliationPaused: false diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 3818f032..d5ca6033 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -63,6 +63,10 @@ pub mod versioned { /// Name of the topic in Kafka. Defaults to `metadata.name`. Immutable. #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "!self.startsWith('__')", + "message": "topics starting with __ are internal to Kafka" + }]))] pub topic_name: Option, /// Number of partitions. Can only be increased. Defaults to the broker setting @@ -274,6 +278,10 @@ mod tests { spec["x-kubernetes-validations"][0]["rule"] ); assert_eq!(249, properties["topicName"]["maxLength"]); + assert_eq!( + "!self.startsWith('__')", + properties["topicName"]["x-kubernetes-validations"][0]["rule"] + ); assert!(properties["topicName"]["pattern"].is_string()); assert_eq!(Some(1.0), properties["partitions"]["minimum"].as_f64()); assert_eq!( From 57b853bb4ea3af7871f1a3a4ced0743961f3ea0d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 18:10:40 +0100 Subject: [PATCH 16/33] fix topic name immutability validation --- Cargo.nix | 20 ++++----- crate-hashes.json | 20 ++++----- extra/crds.yaml | 2 +- rust/shared/src/crd/topic.rs | 5 ++- .../kuttl/topic-authentication/10-assert.yaml | 1 + .../topic-authentication/11-assert.yaml.j2 | 6 ++- .../kuttl/topic-authentication/20-assert.yaml | 19 ++++++-- .../topic-authentication/20-create-topic.yaml | 3 +- .../topic-authentication/topic-condition.sh | 11 ----- tests/templates/kuttl/topic/10-assert.yaml | 1 + tests/templates/kuttl/topic/20-assert.yaml | 40 +++++++++++++---- .../kuttl/topic/20-create-topic.yaml | 5 ++- tests/templates/kuttl/topic/21-assert.yaml | 21 +++++++++ .../kuttl/topic/21-create-defaults-topic.yaml | 10 +++++ .../kuttl/topic/22-reject-internal-topic.yaml | 8 ++++ .../kuttl/topic/22_internal_topic.yaml | 9 ++++ tests/templates/kuttl/topic/30-assert.yaml | 24 +++++++--- .../kuttl/topic/30-change-topic.yaml | 5 ++- .../topic/31-reject-topic-name-change.yaml | 22 ++++++++++ tests/templates/kuttl/topic/41-assert.yaml | 31 +++++++++---- ...ey.yaml => 41-remove-config-override.yaml} | 2 +- tests/templates/kuttl/topic/50-assert.yaml | 19 ++++++-- .../kuttl/topic/50-unsupported-change.yaml | 5 ++- tests/templates/kuttl/topic/51-assert.yaml | 6 --- tests/templates/kuttl/topic/60-assert.yaml | 17 +++++-- .../kuttl/topic/60-invalid-spec.yaml | 5 ++- .../kuttl/topic/70-create-foreign-topic.yaml | 4 +- ...topic.yaml => 71-adopt-foreign-topic.yaml} | 4 +- tests/templates/kuttl/topic/71-assert.yaml | 31 +++++++++++-- .../kuttl/topic/72-adopt-foreign-topic.yaml | 12 ----- tests/templates/kuttl/topic/72-assert.yaml | 8 ---- tests/templates/kuttl/topic/75-assert.yaml | 22 ++++++++++ .../templates/kuttl/topic/75-pause-topic.yaml | 16 +++++++ tests/templates/kuttl/topic/76-assert.yaml | 22 ++++++++++ ...orted-change.yaml => 76-resume-topic.yaml} | 9 ++-- tests/templates/kuttl/topic/77-assert.yaml | 21 +++++++++ .../kuttl/topic/77-invalid-config-value.yaml | 11 +++++ tests/templates/kuttl/topic/80-assert.yaml | 44 ++++++++++++++++--- .../topic/80-create-conflicting-topic.yaml | 4 +- tests/templates/kuttl/topic/81-assert.yaml | 1 + tests/templates/kuttl/topic/90-assert.yaml | 8 ---- .../90-delete-legacy-topic-in-kafka.yaml | 13 ------ .../kuttl/topic/91-delete-topics.yaml | 3 ++ tests/templates/kuttl/topic/92-assert.yaml | 20 +++++++++ .../kuttl/topic/92-create-cleanup-topic.yaml | 13 ++++++ tests/templates/kuttl/topic/95-assert.yaml | 18 ++++++++ tests/templates/kuttl/topic/96-assert.yaml | 15 +++++++ .../topic/96-delete-topic-without-agent.yaml | 6 +++ .../kuttl/topic/97-delete-kafka-cluster.yaml | 7 +++ tests/templates/kuttl/topic/97-errors.yaml | 6 +++ .../templates/kuttl/topic/topic-condition.sh | 11 ----- tests/test-definition.yaml | 2 +- 52 files changed, 496 insertions(+), 152 deletions(-) delete mode 100644 tests/templates/kuttl/topic-authentication/topic-condition.sh create mode 100644 tests/templates/kuttl/topic/21-assert.yaml create mode 100644 tests/templates/kuttl/topic/21-create-defaults-topic.yaml create mode 100644 tests/templates/kuttl/topic/22-reject-internal-topic.yaml create mode 100644 tests/templates/kuttl/topic/22_internal_topic.yaml create mode 100644 tests/templates/kuttl/topic/31-reject-topic-name-change.yaml rename tests/templates/kuttl/topic/{41-remove-config-key.yaml => 41-remove-config-override.yaml} (74%) delete mode 100644 tests/templates/kuttl/topic/51-assert.yaml rename tests/templates/kuttl/topic/{71-refuse-foreign-topic.yaml => 71-adopt-foreign-topic.yaml} (71%) delete mode 100644 tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml delete mode 100644 tests/templates/kuttl/topic/72-assert.yaml create mode 100644 tests/templates/kuttl/topic/75-assert.yaml create mode 100644 tests/templates/kuttl/topic/75-pause-topic.yaml create mode 100644 tests/templates/kuttl/topic/76-assert.yaml rename tests/templates/kuttl/topic/{51-revert-unsupported-change.yaml => 76-resume-topic.yaml} (53%) create mode 100644 tests/templates/kuttl/topic/77-assert.yaml create mode 100644 tests/templates/kuttl/topic/77-invalid-config-value.yaml delete mode 100644 tests/templates/kuttl/topic/90-assert.yaml delete mode 100644 tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml create mode 100644 tests/templates/kuttl/topic/92-assert.yaml create mode 100644 tests/templates/kuttl/topic/92-create-cleanup-topic.yaml create mode 100644 tests/templates/kuttl/topic/95-assert.yaml create mode 100644 tests/templates/kuttl/topic/96-assert.yaml create mode 100644 tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml create mode 100644 tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml create mode 100644 tests/templates/kuttl/topic/97-errors.yaml delete mode 100644 tests/templates/kuttl/topic/topic-condition.sh diff --git a/Cargo.nix b/Cargo.nix index 57339206..c4e4c8ec 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5510,7 +5510,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "k8s_version"; authors = [ @@ -10489,7 +10489,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_agent"; authors = [ @@ -10532,7 +10532,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_certs"; authors = [ @@ -10934,7 +10934,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_operator"; authors = [ @@ -11139,7 +11139,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -11174,7 +11174,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_shared"; authors = [ @@ -11255,7 +11255,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_telemetry"; authors = [ @@ -11365,7 +11365,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_versioned"; authors = [ @@ -11411,7 +11411,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11475,7 +11475,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 017105d5..405952ac 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s" } \ No newline at end of file diff --git a/extra/crds.yaml b/extra/crds.yaml index d7b52284..92407add 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2343,7 +2343,7 @@ spec: type: object x-kubernetes-validations: - message: topicName is immutable - rule: self.?topicName == oldSelf.?topicName + rule: has(self.topicName) == has(oldSelf.topicName) && (!has(self.topicName) || self.topicName == oldSelf.topicName) status: nullable: true properties: diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index d5ca6033..4bf75071 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -50,7 +50,7 @@ pub mod versioned { #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] #[schemars(extend("x-kubernetes-validations" = [{ - "rule": "self.?topicName == oldSelf.?topicName", + "rule": "has(self.topicName) == has(oldSelf.topicName) && (!has(self.topicName) || self.topicName == oldSelf.topicName)", "message": "topicName is immutable" }]))] pub struct KafkaTopicSpec { @@ -274,7 +274,8 @@ mod tests { properties["clusterRef"]["x-kubernetes-validations"][0]["rule"] ); assert_eq!( - "self.?topicName == oldSelf.?topicName", + "has(self.topicName) == has(oldSelf.topicName) && (!has(self.topicName) || \ + self.topicName == oldSelf.topicName)", spec["x-kubernetes-validations"][0]["rule"] ); assert_eq!(249, properties["topicName"]["maxLength"]); diff --git a/tests/templates/kuttl/topic-authentication/10-assert.yaml b/tests/templates/kuttl/topic-authentication/10-assert.yaml index 5b1be639..bc622bea 100644 --- a/tests/templates/kuttl/topic-authentication/10-assert.yaml +++ b/tests/templates/kuttl/topic-authentication/10-assert.yaml @@ -1,4 +1,5 @@ --- +# The KafkaCluster and its agent are running. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 diff --git a/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 b/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 index c7c29981..1853fe47 100644 --- a/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 +++ b/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 @@ -6,9 +6,11 @@ kind: TestAssert timeout: 60 commands: {% if authentication == 'anonymous' %} - - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "discovery tls-kafka" + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "tls-kafka" - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_CLIENT_SECURITY")].value}')" = tls {% else %} - - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "discovery tls-kafka tls-management-access" + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "tls-kafka tls-management-access" - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_CLIENT_SECURITY")].value}')" = mutual-tls {% endif %} + # The headless Service keeps the traffic within the Kubernetes cluster. + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_BOOTSTRAP_SERVERS")].value}')" = "test-kafka-broker-default-headless.$NAMESPACE.svc.cluster.local:9093" diff --git a/tests/templates/kuttl/topic-authentication/20-assert.yaml b/tests/templates/kuttl/topic-authentication/20-assert.yaml index fcdd3b41..7d87800a 100644 --- a/tests/templates/kuttl/topic-authentication/20-assert.yaml +++ b/tests/templates/kuttl/topic-authentication/20-assert.yaml @@ -1,8 +1,19 @@ --- +# The agent authenticates to Kafka and creates the topic. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders Ready True Available - - script: bash topic-condition.sh orders Synced True Synced - - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 3"' +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + partitions: 3 diff --git a/tests/templates/kuttl/topic-authentication/20-create-topic.yaml b/tests/templates/kuttl/topic-authentication/20-create-topic.yaml index a0020413..8a57a0e5 100644 --- a/tests/templates/kuttl/topic-authentication/20-create-topic.yaml +++ b/tests/templates/kuttl/topic-authentication/20-create-topic.yaml @@ -4,7 +4,8 @@ kind: KafkaTopic metadata: name: orders spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 3 replicationFactor: 1 diff --git a/tests/templates/kuttl/topic-authentication/topic-condition.sh b/tests/templates/kuttl/topic-authentication/topic-condition.sh deleted file mode 100644 index 59a05a0d..00000000 --- a/tests/templates/kuttl/topic-authentication/topic-condition.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# Usage: topic-condition.sh -# Fails unless the condition of the KafkaTopic has the given status and reason. -set -euo pipefail - -actual=$(kubectl get kafkatopic --namespace "$NAMESPACE" "$1" \ - --output "jsonpath={.status.conditions[?(@.type==\"$2\")].status}/{.status.conditions[?(@.type==\"$2\")].reason}") -if [ "$actual" != "$3/$4" ]; then - echo "$1: expected the condition $2 to be $3/$4, but it is \"$actual\"" - exit 1 -fi diff --git a/tests/templates/kuttl/topic/10-assert.yaml b/tests/templates/kuttl/topic/10-assert.yaml index dcb6e261..bd9419b2 100644 --- a/tests/templates/kuttl/topic/10-assert.yaml +++ b/tests/templates/kuttl/topic/10-assert.yaml @@ -1,4 +1,5 @@ --- +# The KafkaCluster and its agent are running. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 diff --git a/tests/templates/kuttl/topic/20-assert.yaml b/tests/templates/kuttl/topic/20-assert.yaml index fcc58d2c..d358950b 100644 --- a/tests/templates/kuttl/topic/20-assert.yaml +++ b/tests/templates/kuttl/topic/20-assert.yaml @@ -1,12 +1,36 @@ --- +# With the default deletionPolicy Retain, the KafkaTopic has no finalizer. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders Ready True Available - - script: bash topic-condition.sh orders Synced True Synced - - script: test -n "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.topicId}')" - - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.partitions[*].index}')" = "0 1 2" - - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 3"' - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=3600000" - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "max.message.bytes=2097152" +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders + ref: orders +assertAll: + - celExpr: orders.status.topicId != '' + - celExpr: "!has(orders.metadata.finalizers)" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + topicName: orders + partitions: 3 + replicationFactor: 1 + config: + retention.ms: + value: "3600000" + source: DYNAMIC_TOPIC_CONFIG + max.message.bytes: + value: "2097152" + source: DYNAMIC_TOPIC_CONFIG diff --git a/tests/templates/kuttl/topic/20-create-topic.yaml b/tests/templates/kuttl/topic/20-create-topic.yaml index e91eecfb..d64165b4 100644 --- a/tests/templates/kuttl/topic/20-create-topic.yaml +++ b/tests/templates/kuttl/topic/20-create-topic.yaml @@ -4,10 +4,11 @@ kind: KafkaTopic metadata: name: orders spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 3 replicationFactor: 1 - config: + configOverrides: retention.ms: "3600000" max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/21-assert.yaml b/tests/templates/kuttl/topic/21-assert.yaml new file mode 100644 index 00000000..fa8996d9 --- /dev/null +++ b/tests/templates/kuttl/topic/21-assert.yaml @@ -0,0 +1,21 @@ +--- +# The broker defaults num.partitions and default.replication.factor are both 1. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + topicName: defaults + partitions: 1 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/21-create-defaults-topic.yaml b/tests/templates/kuttl/topic/21-create-defaults-topic.yaml new file mode 100644 index 00000000..bfac2b98 --- /dev/null +++ b/tests/templates/kuttl/topic/21-create-defaults-topic.yaml @@ -0,0 +1,10 @@ +--- +# Without topicName, partitions and replicationFactor: the topic is named after the KafkaTopic and +# gets the broker defaults. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +spec: + clusterRef: + name: test-kafka diff --git a/tests/templates/kuttl/topic/22-reject-internal-topic.yaml b/tests/templates/kuttl/topic/22-reject-internal-topic.yaml new file mode 100644 index 00000000..a2518122 --- /dev/null +++ b/tests/templates/kuttl/topic/22-reject-internal-topic.yaml @@ -0,0 +1,8 @@ +--- +# The API server rejects topics starting with __, which are internal to Kafka. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + kubectl apply --namespace $NAMESPACE --filename 22_internal_topic.yaml 2>&1 + | grep "topics starting with __ are internal to Kafka" diff --git a/tests/templates/kuttl/topic/22_internal_topic.yaml b/tests/templates/kuttl/topic/22_internal_topic.yaml new file mode 100644 index 00000000..8003b850 --- /dev/null +++ b/tests/templates/kuttl/topic/22_internal_topic.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: consumer-offsets +spec: + clusterRef: + name: test-kafka + topicName: __consumer_offsets diff --git a/tests/templates/kuttl/topic/30-assert.yaml b/tests/templates/kuttl/topic/30-assert.yaml index f8eaa137..0f98e197 100644 --- a/tests/templates/kuttl/topic/30-assert.yaml +++ b/tests/templates/kuttl/topic/30-assert.yaml @@ -1,9 +1,23 @@ --- +# The agent adds the partitions and applies the changed config. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders Synced True Synced - - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.partitions[*].index}')" = "0 1 2 3 4" - - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 5"' - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + partitions: 5 + config: + retention.ms: + value: "7200000" + source: DYNAMIC_TOPIC_CONFIG diff --git a/tests/templates/kuttl/topic/30-change-topic.yaml b/tests/templates/kuttl/topic/30-change-topic.yaml index 9b4b2667..ef186c7b 100644 --- a/tests/templates/kuttl/topic/30-change-topic.yaml +++ b/tests/templates/kuttl/topic/30-change-topic.yaml @@ -4,10 +4,11 @@ kind: KafkaTopic metadata: name: orders spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 5 replicationFactor: 1 - config: + configOverrides: retention.ms: "7200000" max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/31-reject-topic-name-change.yaml b/tests/templates/kuttl/topic/31-reject-topic-name-change.yaml new file mode 100644 index 00000000..14d5e3af --- /dev/null +++ b/tests/templates/kuttl/topic/31-reject-topic-name-change.yaml @@ -0,0 +1,22 @@ +--- +# The API server rejects changing, removing or adding topicName. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + kubectl patch kafkatopic orders --namespace $NAMESPACE --type merge + --patch '{"spec": {"topicName": "renamed"}}' 2>&1 + | grep "topicName is immutable" + - script: >- + kubectl patch kafkatopic orders --namespace $NAMESPACE --type json + --patch '[{"op": "remove", "path": "/spec/topicName"}]' 2>&1 + | grep "topicName is immutable" + - script: >- + kubectl patch kafkatopic defaults --namespace $NAMESPACE --type merge + --patch '{"spec": {"topicName": "defaults"}}' 2>&1 + | grep "topicName is immutable" + - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.spec.topicName}')" = orders + # An explicit null means the same as no topicName. + - script: >- + kubectl patch kafkatopic defaults --namespace $NAMESPACE --type json + --patch '[{"op": "add", "path": "/spec/topicName", "value": null}]' diff --git a/tests/templates/kuttl/topic/41-assert.yaml b/tests/templates/kuttl/topic/41-assert.yaml index 796e09f8..2a5e4b77 100644 --- a/tests/templates/kuttl/topic/41-assert.yaml +++ b/tests/templates/kuttl/topic/41-assert.yaml @@ -1,12 +1,27 @@ --- -# The key removed from the spec is reset, the key set out of band stays. +# The configOverrides replace all topic-level configs: the key removed from the spec and the key +# set out of band are both reset to the default. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders Synced True Synced - - script: | - configs=$(bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders) - ! echo "$configs" | grep max.message.bytes - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "segment.ms=86400000" - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + retention.ms: + value: "7200000" + source: DYNAMIC_TOPIC_CONFIG + max.message.bytes: + source: DEFAULT_CONFIG + segment.ms: + source: DEFAULT_CONFIG diff --git a/tests/templates/kuttl/topic/41-remove-config-key.yaml b/tests/templates/kuttl/topic/41-remove-config-override.yaml similarity index 74% rename from tests/templates/kuttl/topic/41-remove-config-key.yaml rename to tests/templates/kuttl/topic/41-remove-config-override.yaml index 88012997..ca178649 100644 --- a/tests/templates/kuttl/topic/41-remove-config-key.yaml +++ b/tests/templates/kuttl/topic/41-remove-config-override.yaml @@ -5,4 +5,4 @@ kind: TestStep commands: - script: >- kubectl patch kafkatopic orders --namespace $NAMESPACE --type json - --patch '[{"op": "remove", "path": "/spec/config/max.message.bytes"}]' + --patch '[{"op": "remove", "path": "/spec/configOverrides/max.message.bytes"}]' diff --git a/tests/templates/kuttl/topic/50-assert.yaml b/tests/templates/kuttl/topic/50-assert.yaml index 443a7e5a..b7945043 100644 --- a/tests/templates/kuttl/topic/50-assert.yaml +++ b/tests/templates/kuttl/topic/50-assert.yaml @@ -3,7 +3,18 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders Synced False UnsupportedChange - - script: bash topic-condition.sh orders Ready True Available - - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 5"' +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "False" + reason: UnsupportedChange + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/50-unsupported-change.yaml b/tests/templates/kuttl/topic/50-unsupported-change.yaml index 72f427bc..9424a96d 100644 --- a/tests/templates/kuttl/topic/50-unsupported-change.yaml +++ b/tests/templates/kuttl/topic/50-unsupported-change.yaml @@ -5,9 +5,10 @@ kind: KafkaTopic metadata: name: orders spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 2 replicationFactor: 3 - config: + configOverrides: retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/51-assert.yaml b/tests/templates/kuttl/topic/51-assert.yaml deleted file mode 100644 index 3f2cfc72..00000000 --- a/tests/templates/kuttl/topic/51-assert.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -commands: - - script: bash topic-condition.sh orders Synced True Synced diff --git a/tests/templates/kuttl/topic/60-assert.yaml b/tests/templates/kuttl/topic/60-assert.yaml index d212ade8..a2ae21b5 100644 --- a/tests/templates/kuttl/topic/60-assert.yaml +++ b/tests/templates/kuttl/topic/60-assert.yaml @@ -1,11 +1,22 @@ --- -# Ready stays unknown, because the agent did not look at the topic in Kafka. +# Ready is unknown, because the agent did not look at the topic in Kafka. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 commands: - - script: bash topic-condition.sh throttled Synced False InvalidSpec - - script: bash topic-condition.sh throttled Ready "" "" - script: | topics=$(bash kafka-cli.sh kafka-topics.sh --list) ! echo "$topics" | grep -x throttled +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: throttled +status: + conditions: + - type: Ready + status: Unknown + reason: InvalidSpec + - type: Synced + status: "False" + reason: InvalidSpec diff --git a/tests/templates/kuttl/topic/60-invalid-spec.yaml b/tests/templates/kuttl/topic/60-invalid-spec.yaml index effcf8bc..e331d366 100644 --- a/tests/templates/kuttl/topic/60-invalid-spec.yaml +++ b/tests/templates/kuttl/topic/60-invalid-spec.yaml @@ -5,9 +5,10 @@ kind: KafkaTopic metadata: name: throttled spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: throttled partitions: 1 replicationFactor: 1 - config: + configOverrides: leader.replication.throttled.replicas: "*" diff --git a/tests/templates/kuttl/topic/70-create-foreign-topic.yaml b/tests/templates/kuttl/topic/70-create-foreign-topic.yaml index da72f46a..9591c9ca 100644 --- a/tests/templates/kuttl/topic/70-create-foreign-topic.yaml +++ b/tests/templates/kuttl/topic/70-create-foreign-topic.yaml @@ -3,4 +3,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: bash kafka-cli.sh kafka-topics.sh --create --topic legacy --partitions 1 --replication-factor 1 + - script: >- + bash kafka-cli.sh kafka-topics.sh --create --topic legacy --partitions 1 --replication-factor 1 + --config cleanup.policy=compact diff --git a/tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml b/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml similarity index 71% rename from tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml rename to tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml index fe7eef33..71b6bedf 100644 --- a/tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml +++ b/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml @@ -4,7 +4,9 @@ kind: KafkaTopic metadata: name: legacy spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: legacy partitions: 1 replicationFactor: 1 + deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/71-assert.yaml b/tests/templates/kuttl/topic/71-assert.yaml index 3ed15c31..2a319889 100644 --- a/tests/templates/kuttl/topic/71-assert.yaml +++ b/tests/templates/kuttl/topic/71-assert.yaml @@ -1,8 +1,31 @@ --- -# The adoption policy defaults to Refuse. +# Existing topics are adopted, and their topic-level configs are replaced. With the deletionPolicy +# Delete, the adopted topic gets a finalizer. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh legacy Ready False AlreadyExists - - script: test -z "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.status.topicId}')" +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: legacy + ref: legacy +assertAll: + - celExpr: legacy.status.topicId != '' +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: legacy + finalizers: + - kafka.stackable.tech/kafkatopic +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + cleanup.policy: + source: DEFAULT_CONFIG diff --git a/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml b/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml deleted file mode 100644 index ef3a851d..00000000 --- a/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaTopic -metadata: - name: legacy -spec: - clusterName: test-kafka - topicName: legacy - partitions: 1 - replicationFactor: 1 - managementPolicy: - adoption: Adopt diff --git a/tests/templates/kuttl/topic/72-assert.yaml b/tests/templates/kuttl/topic/72-assert.yaml deleted file mode 100644 index 02e00c9f..00000000 --- a/tests/templates/kuttl/topic/72-assert.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -commands: - - script: bash topic-condition.sh legacy Ready True Available - - script: bash topic-condition.sh legacy Synced True Synced - - script: test -n "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.metadata.annotations.stackable\.tech/owned-since}')" diff --git a/tests/templates/kuttl/topic/75-assert.yaml b/tests/templates/kuttl/topic/75-assert.yaml new file mode 100644 index 00000000..04bb2d1f --- /dev/null +++ b/tests/templates/kuttl/topic/75-assert.yaml @@ -0,0 +1,22 @@ +--- +# The status still shows the topic in Kafka, with the old config. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "False" + reason: ReconciliationPaused + config: + retention.ms: + value: "7200000" + source: DYNAMIC_TOPIC_CONFIG diff --git a/tests/templates/kuttl/topic/75-pause-topic.yaml b/tests/templates/kuttl/topic/75-pause-topic.yaml new file mode 100644 index 00000000..ea35cd52 --- /dev/null +++ b/tests/templates/kuttl/topic/75-pause-topic.yaml @@ -0,0 +1,16 @@ +--- +# The agent only observes a paused topic, so the changed config is not applied. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterRef: + name: test-kafka + topicName: orders + partitions: 5 + replicationFactor: 1 + configOverrides: + retention.ms: "10800000" + topicOperation: + reconciliationPaused: true diff --git a/tests/templates/kuttl/topic/76-assert.yaml b/tests/templates/kuttl/topic/76-assert.yaml new file mode 100644 index 00000000..08177dc2 --- /dev/null +++ b/tests/templates/kuttl/topic/76-assert.yaml @@ -0,0 +1,22 @@ +--- +# After resuming, the agent applies the config changed while paused. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + retention.ms: + value: "10800000" + source: DYNAMIC_TOPIC_CONFIG diff --git a/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml b/tests/templates/kuttl/topic/76-resume-topic.yaml similarity index 53% rename from tests/templates/kuttl/topic/51-revert-unsupported-change.yaml rename to tests/templates/kuttl/topic/76-resume-topic.yaml index 4a1fa199..8cc896dc 100644 --- a/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml +++ b/tests/templates/kuttl/topic/76-resume-topic.yaml @@ -4,9 +4,12 @@ kind: KafkaTopic metadata: name: orders spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 5 replicationFactor: 1 - config: - retention.ms: "7200000" + configOverrides: + retention.ms: "10800000" + topicOperation: + reconciliationPaused: false diff --git a/tests/templates/kuttl/topic/77-assert.yaml b/tests/templates/kuttl/topic/77-assert.yaml new file mode 100644 index 00000000..26c91f18 --- /dev/null +++ b/tests/templates/kuttl/topic/77-assert.yaml @@ -0,0 +1,21 @@ +--- +# The topic stays available, and the status keeps the config observed in Kafka. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "False" + reason: KafkaRejected + config: + retention.ms: + source: DEFAULT_CONFIG diff --git a/tests/templates/kuttl/topic/77-invalid-config-value.yaml b/tests/templates/kuttl/topic/77-invalid-config-value.yaml new file mode 100644 index 00000000..a230cce9 --- /dev/null +++ b/tests/templates/kuttl/topic/77-invalid-config-value.yaml @@ -0,0 +1,11 @@ +--- +# Kafka rejects the config value, which only it can validate. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +spec: + clusterRef: + name: test-kafka + configOverrides: + retention.ms: "abc" diff --git a/tests/templates/kuttl/topic/80-assert.yaml b/tests/templates/kuttl/topic/80-assert.yaml index ab8423f0..20b4780b 100644 --- a/tests/templates/kuttl/topic/80-assert.yaml +++ b/tests/templates/kuttl/topic/80-assert.yaml @@ -1,9 +1,43 @@ --- -# The older KafkaTopic keeps managing the topic. +# The older KafkaTopic keeps managing the topic. The newer one gets no finalizer, so deleting it +# does not delete the topic. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders-copy Ready False Conflict - - script: bash topic-condition.sh orders Ready True Available - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders-copy + ref: orders_copy +assertAll: + - celExpr: "!has(orders_copy.metadata.finalizers)" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders-copy +status: + conditions: + - type: Ready + status: "False" + reason: Conflict + - type: Synced + status: "False" + reason: Conflict +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + retention.ms: + value: "10800000" + source: DYNAMIC_TOPIC_CONFIG diff --git a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml index 3a2b28c5..7314dcc8 100644 --- a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml +++ b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml @@ -5,7 +5,9 @@ kind: KafkaTopic metadata: name: orders-copy spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 5 replicationFactor: 1 + deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/81-assert.yaml b/tests/templates/kuttl/topic/81-assert.yaml index 091073c7..1e7d88dc 100644 --- a/tests/templates/kuttl/topic/81-assert.yaml +++ b/tests/templates/kuttl/topic/81-assert.yaml @@ -1,4 +1,5 @@ --- +# Deleting the conflicting KafkaTopic leaves the topic in Kafka. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 diff --git a/tests/templates/kuttl/topic/90-assert.yaml b/tests/templates/kuttl/topic/90-assert.yaml deleted file mode 100644 index b2bebc87..00000000 --- a/tests/templates/kuttl/topic/90-assert.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# Only KafkaTopics that delete their topic in Kafka have a finalizer. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -commands: - - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.metadata.finalizers}')" = '["kafka.stackable.tech/kafkatopic"]' - - script: test -z "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.metadata.finalizers}')" diff --git a/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml b/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml deleted file mode 100644 index 2ea201fe..00000000 --- a/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaTopic -metadata: - name: legacy -spec: - clusterName: test-kafka - topicName: legacy - partitions: 1 - replicationFactor: 1 - managementPolicy: - adoption: Adopt - deletion: Delete diff --git a/tests/templates/kuttl/topic/91-delete-topics.yaml b/tests/templates/kuttl/topic/91-delete-topics.yaml index 430f5841..5de81d40 100644 --- a/tests/templates/kuttl/topic/91-delete-topics.yaml +++ b/tests/templates/kuttl/topic/91-delete-topics.yaml @@ -11,3 +11,6 @@ delete: - apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaTopic name: throttled + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: defaults diff --git a/tests/templates/kuttl/topic/92-assert.yaml b/tests/templates/kuttl/topic/92-assert.yaml new file mode 100644 index 00000000..64caeabb --- /dev/null +++ b/tests/templates/kuttl/topic/92-assert.yaml @@ -0,0 +1,20 @@ +--- +# While the agent is still running, it creates the topic and adds the finalizer. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup + finalizers: + - kafka.stackable.tech/kafkatopic +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced diff --git a/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml b/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml new file mode 100644 index 00000000..f14412ee --- /dev/null +++ b/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml @@ -0,0 +1,13 @@ +--- +# From here on, the operator takes over the KafkaTopic, because the agent is removed in the +# following steps. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup +spec: + clusterRef: + name: test-kafka + partitions: 1 + replicationFactor: 1 + deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/95-assert.yaml b/tests/templates/kuttl/topic/95-assert.yaml new file mode 100644 index 00000000..9df1148e --- /dev/null +++ b/tests/templates/kuttl/topic/95-assert.yaml @@ -0,0 +1,18 @@ +--- +# Without an agent, the operator sets the conditions of the remaining KafkaTopics. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup +status: + conditions: + - type: Ready + status: Unknown + reason: AgentUnavailable + - type: Synced + status: "False" + reason: AgentUnavailable diff --git a/tests/templates/kuttl/topic/96-assert.yaml b/tests/templates/kuttl/topic/96-assert.yaml new file mode 100644 index 00000000..d99243f8 --- /dev/null +++ b/tests/templates/kuttl/topic/96-assert.yaml @@ -0,0 +1,15 @@ +--- +# The operator keeps the finalizer while the KafkaCluster exists, so the deletion waits for an +# agent. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: sleep 30 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: cleanup + ref: cleanup +assertAll: + - celExpr: has(cleanup.metadata.deletionTimestamp) diff --git a/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml b/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml new file mode 100644 index 00000000..57614fa4 --- /dev/null +++ b/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml @@ -0,0 +1,6 @@ +--- +# kuttl would wait for the deletion, which only finishes with the KafkaCluster. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl delete kafkatopic cleanup --namespace $NAMESPACE --wait=false diff --git a/tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml b/tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml new file mode 100644 index 00000000..039bc529 --- /dev/null +++ b/tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + name: test-kafka diff --git a/tests/templates/kuttl/topic/97-errors.yaml b/tests/templates/kuttl/topic/97-errors.yaml new file mode 100644 index 00000000..4e62a73a --- /dev/null +++ b/tests/templates/kuttl/topic/97-errors.yaml @@ -0,0 +1,6 @@ +--- +# Without the KafkaCluster, the operator removes the finalizer, so the deletion finishes. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup diff --git a/tests/templates/kuttl/topic/topic-condition.sh b/tests/templates/kuttl/topic/topic-condition.sh deleted file mode 100644 index 59a05a0d..00000000 --- a/tests/templates/kuttl/topic/topic-condition.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# Usage: topic-condition.sh -# Fails unless the condition of the KafkaTopic has the given status and reason. -set -euo pipefail - -actual=$(kubectl get kafkatopic --namespace "$NAMESPACE" "$1" \ - --output "jsonpath={.status.conditions[?(@.type==\"$2\")].status}/{.status.conditions[?(@.type==\"$2\")].reason}") -if [ "$actual" != "$3/$4" ]; then - echo "$1: expected the condition $2 to be $3/$4, but it is \"$actual\"" - exit 1 -fi diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 5c9116ac..85ac9136 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -7,7 +7,7 @@ dimensions: - name: kafka-kraft values: - 3.9.2 - # - 4.3.1 + - 4.3.1 - name: kafka values: - 3.9.2 From 41f8b3708848d01b0d00b2d2655df914169583db Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 10:15:35 +0100 Subject: [PATCH 17/33] change controller and CRD to use status.config without source --- extra/crds.yaml | 14 +-- rust/agent-binary/src/topic/condition.rs | 2 +- rust/agent-binary/src/topic/plan.rs | 95 +++++++------------ rust/agent-binary/src/topic/sync.rs | 15 ++- rust/agent-binary/src/topic/update_status.rs | 32 +------ rust/shared/src/crd/topic.rs | 15 +-- .../00-patch-ns.yaml.j2 | 0 .../05-create-agent-credential.yaml.j2 | 0 .../10-assert.yaml | 0 .../10-install-kafka.yaml | 0 .../10_kafka.yaml.j2 | 0 .../11-assert.yaml.j2 | 0 .../20-assert.yaml | 0 .../20-create-topic.yaml | 0 .../kafka-cli.sh | 0 tests/templates/kuttl/topic/20-assert.yaml | 8 +- tests/templates/kuttl/topic/30-assert.yaml | 4 +- tests/templates/kuttl/topic/41-assert.yaml | 16 ++-- tests/templates/kuttl/topic/71-assert.yaml | 4 +- tests/templates/kuttl/topic/75-assert.yaml | 4 +- tests/templates/kuttl/topic/76-assert.yaml | 4 +- tests/templates/kuttl/topic/77-assert.yaml | 12 ++- tests/templates/kuttl/topic/80-assert.yaml | 4 +- tests/test-definition.yaml | 18 ++-- 24 files changed, 85 insertions(+), 162 deletions(-) rename tests/templates/kuttl/{topic-authentication => management-access}/00-patch-ns.yaml.j2 (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/05-create-agent-credential.yaml.j2 (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/10-assert.yaml (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/10-install-kafka.yaml (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/10_kafka.yaml.j2 (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/11-assert.yaml.j2 (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/20-assert.yaml (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/20-create-topic.yaml (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/kafka-cli.sh (100%) diff --git a/extra/crds.yaml b/extra/crds.yaml index 92407add..8e6f437c 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2383,19 +2383,9 @@ spec: type: array config: additionalProperties: - properties: - source: - description: Where the value comes from, like `DYNAMIC_TOPIC_CONFIG` or `DEFAULT_CONFIG`. - type: string - value: - description: The value, not set for sensitive configs. - nullable: true - type: string - required: - - source - type: object + type: string default: {} - description: All effective topic configs in Kafka. + description: The configs set on the topic in Kafka, without broker and default values. type: object partitions: description: The number of partitions in Kafka. diff --git a/rust/agent-binary/src/topic/condition.rs b/rust/agent-binary/src/topic/condition.rs index 46ed4f80..ccdba66b 100644 --- a/rust/agent-binary/src/topic/condition.rs +++ b/rust/agent-binary/src/topic/condition.rs @@ -213,7 +213,7 @@ mod tests { id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), partition_count, replication_factor: Some(1), - effective_configs: BTreeMap::new(), + topic_configs: BTreeMap::new(), } } diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs index a52b21ec..ca70fca8 100644 --- a/rust/agent-binary/src/topic/plan.rs +++ b/rust/agent-binary/src/topic/plan.rs @@ -2,9 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use stackable_kafka_admin::client::{ - ConfigSource, CreatedTopic, PartitionReassignment, Topic, TopicConfig, -}; +use stackable_kafka_admin::client::{CreatedTopic, PartitionReassignment, Topic}; use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopicSpec}; /// Set by `kafka-reassign-partitions` and Cruise Control for the duration of a reassignment. @@ -17,7 +15,7 @@ pub const THROTTLE_CONFIG_KEYS: [&str; 2] = [ /// The state of an existing topic in Kafka. pub struct ObservedTopic { pub topic: Topic, - pub effective_configs: BTreeMap, + pub topic_configs: BTreeMap, pub reassignments: Vec, } @@ -28,7 +26,7 @@ pub struct TopicState { pub id: String, pub partition_count: i32, pub replication_factor: Option, - pub effective_configs: BTreeMap, + pub topic_configs: BTreeMap, } impl TopicState { @@ -38,7 +36,7 @@ impl TopicState { id: observed.topic.id.clone(), partition_count: observed.topic.partitions.len() as i32, replication_factor: replication_factor(observed), - effective_configs: observed.effective_configs.clone(), + topic_configs: observed.topic_configs.clone(), } } @@ -48,7 +46,7 @@ impl TopicState { id: created.id, partition_count: created.partitions, replication_factor: Some(created.replication_factor), - effective_configs: created.configs, + topic_configs: created.configs, } } } @@ -107,7 +105,7 @@ pub fn plan(spec: &KafkaTopicSpec, observed: Option<&ObservedTopic>) -> TopicPla let actual_partition_count = observed.topic.partitions.len() as i32; let (configs_to_set, configs_to_reset) = - config_changes(&spec.config_overrides, &observed.effective_configs); + config_changes(&spec.config_overrides, &observed.topic_configs); TopicPlan::Update { topic_update: TopicUpdate { @@ -153,29 +151,27 @@ fn unsupported_changes(spec: &KafkaTopicSpec, observed: &ObservedTopic) -> Vec, - effective_configs: &BTreeMap, + desired_topic_configs: &BTreeMap, + current_topic_configs: &BTreeMap, ) -> (BTreeMap, BTreeSet) { - // Topic configs have no PASSWORD type, so Kafka does not redact their topic-level values. - let topic_level_configs = effective_configs - .iter() - .filter(|(_, config)| config.source == ConfigSource::DynamicTopic) - .map(|(key, config)| (key, config.value.as_deref())) - .collect::>(); - - // Compared with the topic-level value, not the effective one: a desired value that equals the - // broker config is still pinned on the topic. Otherwise a later change of the broker config - // would apply to the topic until the next resync. Unchanged values are skipped, so a resync + // A desired value that equals the broker config is still set on the topic, so a later change of + // the broker config does not apply to the topic. Unchanged values are skipped, so a resync // without drift does not write to Kafka. - let configs_to_set = desired + let configs_to_set = desired_topic_configs .iter() - .filter(|(key, value)| topic_level_configs.get(key) != Some(&Some(value.as_str()))) + .filter(|(key, value)| { + current_topic_configs.get(*key) != Some(value) + && !THROTTLE_CONFIG_KEYS.contains(&key.as_str()) + }) .map(|(key, value)| (key.clone(), value.clone())) .collect(); - let configs_to_reset = topic_level_configs + let configs_to_reset = current_topic_configs .keys() - .filter(|key| !desired.contains_key(**key) && !THROTTLE_CONFIG_KEYS.contains(&key.as_str())) - .map(|key| key.to_string()) + .filter(|key| { + !desired_topic_configs.contains_key(*key) + && !THROTTLE_CONFIG_KEYS.contains(&key.as_str()) + }) + .cloned() .collect(); (configs_to_set, configs_to_reset) } @@ -243,21 +239,6 @@ mod tests { keys.iter().map(|key| key.to_string()).collect() } - fn topic_configs(entries: &[(&str, &str, ConfigSource)]) -> BTreeMap { - entries - .iter() - .map(|(key, value, source)| { - ( - key.to_string(), - TopicConfig { - value: Some(value.to_string()), - source: *source, - }, - ) - }) - .collect() - } - fn update(topic_update: TopicUpdate, unsupported_changes: Vec) -> TopicPlan { TopicPlan::Update { topic_update, @@ -265,8 +246,9 @@ mod tests { } } - /// A topic whose partitions have the given number of replicas - fn observed(replicas: &[usize], configs: &[(&str, &str, ConfigSource)]) -> ObservedTopic { + /// A topic whose partitions have the given number of replicas, with the given topic-level + /// configs + fn observed(replicas: &[usize], topic_configs: &[(&str, &str)]) -> ObservedTopic { ObservedTopic { topic: Topic { id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), @@ -279,7 +261,7 @@ mod tests { }) .collect(), }, - effective_configs: topic_configs(configs), + topic_configs: configs(topic_configs), reassignments: vec![], } } @@ -300,10 +282,7 @@ mod tests { #[test] fn do_nothing_for_a_matching_topic() { - let observed = observed( - &[2, 2], - &[("retention.ms", "1000", ConfigSource::DynamicTopic)], - ); + let observed = observed(&[2, 2], &[("retention.ms", "1000")]); let spec = spec(Some(2), Some(2), &[("retention.ms", "1000")]); assert_eq!( @@ -364,15 +343,9 @@ mod tests { let observed = observed( &[1], &[ - ("retention.ms", "1000", ConfigSource::DynamicTopic), - ("cleanup.policy", "compact", ConfigSource::DynamicTopic), - ("segment.bytes", "1024", ConfigSource::Default), - ("max.message.bytes", "10", ConfigSource::StaticBroker), - ( - "leader.replication.throttled.replicas", - "0:1", - ConfigSource::DynamicTopic, - ), + ("retention.ms", "1000"), + ("cleanup.policy", "compact"), + ("leader.replication.throttled.replicas", "0:1"), ], ); @@ -380,7 +353,11 @@ mod tests { &spec( None, None, - &[("retention.ms", "2000"), ("max.message.bytes", "10")], + &[ + ("retention.ms", "2000"), + ("max.message.bytes", "10"), + ("follower.replication.throttled.replicas", "*"), + ], ), Some(&observed), ); @@ -388,13 +365,11 @@ mod tests { assert_eq!( update( TopicUpdate { - // `max.message.bytes` already has the value, but not on the topic configs_to_set: configs(&[ ("max.message.bytes", "10"), ("retention.ms", "2000") ]), - // `segment.bytes` is not set on the topic, the throttle belongs to a - // reassignment + // The throttles belong to a reassignment configs_to_reset: keys(&["cleanup.policy"]), ..TopicUpdate::default() }, diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs index fd34efb0..97f98028 100644 --- a/rust/agent-binary/src/topic/sync.rs +++ b/rust/agent-binary/src/topic/sync.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use snafu::ResultExt; -use stackable_kafka_admin::client::{KafkaAdminClient, TopicConfig}; +use stackable_kafka_admin::client::KafkaAdminClient; use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopic}; use stackable_operator::kube::ResourceExt; use tracing::info; @@ -86,7 +86,7 @@ async fn observe( .map(|partition| partition.index) .collect::>(); Ok(Some(ObservedTopic { - effective_configs: admin_client + topic_configs: admin_client .describe_topic_configs(topic_name) .await .context(KafkaSnafu)?, @@ -136,7 +136,7 @@ async fn execute( // Config if !topic_update.configs_to_set.is_empty() || !topic_update.configs_to_reset.is_empty() { - let effective_configs = replace_configs( + let topic_configs = replace_configs( kafka_topic, admin_client, &topic_name, @@ -145,7 +145,7 @@ async fn execute( ) .await?; if let Some(topic_state) = topic_state { - topic_state.effective_configs = effective_configs; + topic_state.topic_configs = topic_configs; } } } @@ -204,16 +204,15 @@ async fn grow_partitions( Ok(()) } -/// Returns the effective configs of the topic after the change. They are described again, -/// because IncrementalAlterConfigs does not return them and a reset key falls back to a broker -/// config the agent does not know. +/// Returns the topic-level configs after the change, described again because +/// IncrementalAlterConfigs does not return them. async fn replace_configs( kafka_topic: &KafkaTopic, admin_client: &mut KafkaAdminClient, topic_name: &TopicName, configs_to_set: &BTreeMap, configs_to_reset: &BTreeSet, -) -> Result> { +) -> Result> { admin_client .incremental_alter_configs(topic_name.as_ref(), configs_to_set, configs_to_reset) .await diff --git a/rust/agent-binary/src/topic/update_status.rs b/rust/agent-binary/src/topic/update_status.rs index bff07684..b7fdcfae 100644 --- a/rust/agent-binary/src/topic/update_status.rs +++ b/rust/agent-binary/src/topic/update_status.rs @@ -1,9 +1,7 @@ //! The status of a `KafkaTopic` for the outcome of a sync. use snafu::ResultExt; -use stackable_kafka_shared::crd::topic::v1alpha1::{ - KafkaTopic, KafkaTopicConfigStatus, KafkaTopicStatus, -}; +use stackable_kafka_shared::crd::topic::v1alpha1::{KafkaTopic, KafkaTopicStatus}; use stackable_operator::{ agent::condition::update_conditions, k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, @@ -62,17 +60,7 @@ fn status_from( topic_name: Some(topic_state.name.clone()), partitions: Some(topic_state.partition_count), replication_factor: topic_state.replication_factor, - config: topic_state - .effective_configs - .iter() - .map(|(key, config)| { - let config = KafkaTopicConfigStatus { - value: config.value.clone(), - source: config.source.to_string(), - }; - (key.clone(), config) - }) - .collect(), + config: topic_state.topic_configs.clone(), }, // The topic does not exist or is managed by another KafkaTopic. (TopicSyncOutcome::Conflict { .. } | TopicSyncOutcome::ReconciliationPaused, None) => { @@ -92,7 +80,6 @@ fn status_from( mod tests { use std::{collections::BTreeMap, str::FromStr}; - use stackable_kafka_admin::client::{ConfigSource, TopicConfig}; use stackable_kafka_shared::crd::topic::TopicName; use super::*; @@ -107,13 +94,7 @@ mod tests { id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), partition_count: 6, replication_factor: Some(3), - effective_configs: BTreeMap::from([( - "retention.ms".to_owned(), - TopicConfig { - value: Some("1000".to_owned()), - source: ConfigSource::DynamicTopic, - }, - )]), + topic_configs: BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]), } } @@ -137,11 +118,8 @@ mod tests { assert_eq!(Some(6), status.partitions); assert_eq!(Some(3), status.replication_factor); assert_eq!( - Some(&KafkaTopicConfigStatus { - value: Some("1000".to_owned()), - source: "DYNAMIC_TOPIC_CONFIG".to_owned(), - }), - status.config.get("retention.ms") + Some("1000"), + status.config.get("retention.ms").map(String::as_str) ); assert!( status diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 4bf75071..d2816c97 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -126,20 +126,9 @@ pub mod versioned { #[serde(default, skip_serializing_if = "Option::is_none")] pub replication_factor: Option, - /// All effective topic configs in Kafka. + /// The configs set on the topic in Kafka, without broker and default values. #[serde(default)] - pub config: BTreeMap, - } - - #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] - #[serde(rename_all = "camelCase")] - pub struct KafkaTopicConfigStatus { - /// The value, not set for sensitive configs. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub value: Option, - - /// Where the value comes from, like `DYNAMIC_TOPIC_CONFIG` or `DEFAULT_CONFIG`. - pub source: String, + pub config: BTreeMap, } } diff --git a/tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 b/tests/templates/kuttl/management-access/00-patch-ns.yaml.j2 similarity index 100% rename from tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 rename to tests/templates/kuttl/management-access/00-patch-ns.yaml.j2 diff --git a/tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 b/tests/templates/kuttl/management-access/05-create-agent-credential.yaml.j2 similarity index 100% rename from tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 rename to tests/templates/kuttl/management-access/05-create-agent-credential.yaml.j2 diff --git a/tests/templates/kuttl/topic-authentication/10-assert.yaml b/tests/templates/kuttl/management-access/10-assert.yaml similarity index 100% rename from tests/templates/kuttl/topic-authentication/10-assert.yaml rename to tests/templates/kuttl/management-access/10-assert.yaml diff --git a/tests/templates/kuttl/topic-authentication/10-install-kafka.yaml b/tests/templates/kuttl/management-access/10-install-kafka.yaml similarity index 100% rename from tests/templates/kuttl/topic-authentication/10-install-kafka.yaml rename to tests/templates/kuttl/management-access/10-install-kafka.yaml diff --git a/tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 similarity index 100% rename from tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 rename to tests/templates/kuttl/management-access/10_kafka.yaml.j2 diff --git a/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 b/tests/templates/kuttl/management-access/11-assert.yaml.j2 similarity index 100% rename from tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 rename to tests/templates/kuttl/management-access/11-assert.yaml.j2 diff --git a/tests/templates/kuttl/topic-authentication/20-assert.yaml b/tests/templates/kuttl/management-access/20-assert.yaml similarity index 100% rename from tests/templates/kuttl/topic-authentication/20-assert.yaml rename to tests/templates/kuttl/management-access/20-assert.yaml diff --git a/tests/templates/kuttl/topic-authentication/20-create-topic.yaml b/tests/templates/kuttl/management-access/20-create-topic.yaml similarity index 100% rename from tests/templates/kuttl/topic-authentication/20-create-topic.yaml rename to tests/templates/kuttl/management-access/20-create-topic.yaml diff --git a/tests/templates/kuttl/topic-authentication/kafka-cli.sh b/tests/templates/kuttl/management-access/kafka-cli.sh similarity index 100% rename from tests/templates/kuttl/topic-authentication/kafka-cli.sh rename to tests/templates/kuttl/management-access/kafka-cli.sh diff --git a/tests/templates/kuttl/topic/20-assert.yaml b/tests/templates/kuttl/topic/20-assert.yaml index d358950b..7290dfc0 100644 --- a/tests/templates/kuttl/topic/20-assert.yaml +++ b/tests/templates/kuttl/topic/20-assert.yaml @@ -28,9 +28,5 @@ status: partitions: 3 replicationFactor: 1 config: - retention.ms: - value: "3600000" - source: DYNAMIC_TOPIC_CONFIG - max.message.bytes: - value: "2097152" - source: DYNAMIC_TOPIC_CONFIG + retention.ms: "3600000" + max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/30-assert.yaml b/tests/templates/kuttl/topic/30-assert.yaml index 0f98e197..7d407115 100644 --- a/tests/templates/kuttl/topic/30-assert.yaml +++ b/tests/templates/kuttl/topic/30-assert.yaml @@ -18,6 +18,4 @@ status: reason: Synced partitions: 5 config: - retention.ms: - value: "7200000" - source: DYNAMIC_TOPIC_CONFIG + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/41-assert.yaml b/tests/templates/kuttl/topic/41-assert.yaml index 2a5e4b77..5544c145 100644 --- a/tests/templates/kuttl/topic/41-assert.yaml +++ b/tests/templates/kuttl/topic/41-assert.yaml @@ -4,6 +4,14 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders + ref: orders +assertAll: + - celExpr: "!('max.message.bytes' in orders.status.config)" + - celExpr: "!('segment.ms' in orders.status.config)" --- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaTopic @@ -18,10 +26,4 @@ status: status: "True" reason: Synced config: - retention.ms: - value: "7200000" - source: DYNAMIC_TOPIC_CONFIG - max.message.bytes: - source: DEFAULT_CONFIG - segment.ms: - source: DEFAULT_CONFIG + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/71-assert.yaml b/tests/templates/kuttl/topic/71-assert.yaml index 2a319889..6da767ba 100644 --- a/tests/templates/kuttl/topic/71-assert.yaml +++ b/tests/templates/kuttl/topic/71-assert.yaml @@ -11,6 +11,7 @@ resourceRefs: ref: legacy assertAll: - celExpr: legacy.status.topicId != '' + - celExpr: "!has(legacy.status.config) || !('cleanup.policy' in legacy.status.config)" --- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaTopic @@ -26,6 +27,3 @@ status: - type: Synced status: "True" reason: Synced - config: - cleanup.policy: - source: DEFAULT_CONFIG diff --git a/tests/templates/kuttl/topic/75-assert.yaml b/tests/templates/kuttl/topic/75-assert.yaml index 04bb2d1f..6a856dcf 100644 --- a/tests/templates/kuttl/topic/75-assert.yaml +++ b/tests/templates/kuttl/topic/75-assert.yaml @@ -17,6 +17,4 @@ status: status: "False" reason: ReconciliationPaused config: - retention.ms: - value: "7200000" - source: DYNAMIC_TOPIC_CONFIG + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/76-assert.yaml b/tests/templates/kuttl/topic/76-assert.yaml index 08177dc2..b82d73e2 100644 --- a/tests/templates/kuttl/topic/76-assert.yaml +++ b/tests/templates/kuttl/topic/76-assert.yaml @@ -17,6 +17,4 @@ status: status: "True" reason: Synced config: - retention.ms: - value: "10800000" - source: DYNAMIC_TOPIC_CONFIG + retention.ms: "10800000" diff --git a/tests/templates/kuttl/topic/77-assert.yaml b/tests/templates/kuttl/topic/77-assert.yaml index 26c91f18..e60dcb2c 100644 --- a/tests/templates/kuttl/topic/77-assert.yaml +++ b/tests/templates/kuttl/topic/77-assert.yaml @@ -1,8 +1,15 @@ --- -# The topic stays available, and the status keeps the config observed in Kafka. +# The topic stays available, and the rejected value is not set on the topic. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: defaults + ref: defaults +assertAll: + - celExpr: "!has(defaults.status.config) || !('retention.ms' in defaults.status.config)" --- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaTopic @@ -16,6 +23,3 @@ status: - type: Synced status: "False" reason: KafkaRejected - config: - retention.ms: - source: DEFAULT_CONFIG diff --git a/tests/templates/kuttl/topic/80-assert.yaml b/tests/templates/kuttl/topic/80-assert.yaml index 20b4780b..c22dd5c0 100644 --- a/tests/templates/kuttl/topic/80-assert.yaml +++ b/tests/templates/kuttl/topic/80-assert.yaml @@ -38,6 +38,4 @@ status: status: "True" reason: Synced config: - retention.ms: - value: "10800000" - source: DYNAMIC_TOPIC_CONFIG + retention.ms: "10800000" diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 85ac9136..c3873d45 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -87,15 +87,6 @@ tests: dimensions: - kafka-kraft - openshift - - name: topic - dimensions: - - kafka-kraft - - openshift - - name: topic-authentication - dimensions: - - kafka-kraft - - management-access-authentication - - openshift - name: smoke dimensions: - kafka @@ -162,6 +153,15 @@ tests: - openshift - broker-listener-class - bootstrap-listener-class + - name: topic + dimensions: + - kafka-kraft + - openshift + - name: management-access + dimensions: + - kafka-kraft + - management-access-authentication + - openshift suites: - name: nightly From f1946f7b87188256453459b459bdca7eb91ce0b2 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 11:59:24 +0100 Subject: [PATCH 18/33] add grace period before operator takes over topic status --- rust/operator-binary/src/topic_controller.rs | 190 ++++++++++++++++--- 1 file changed, 162 insertions(+), 28 deletions(-) diff --git a/rust/operator-binary/src/topic_controller.rs b/rust/operator-binary/src/topic_controller.rs index e724e2a9..b7492c94 100644 --- a/rust/operator-binary/src/topic_controller.rs +++ b/rust/operator-binary/src/topic_controller.rs @@ -13,7 +13,9 @@ use stackable_operator::{ ConditionStatus, ManagedCondition, ManagedConditionType, update_conditions, }, k8s_openapi::{ - api::apps::v1::Deployment, apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp, + api::apps::v1::Deployment, + apimachinery::pkg::apis::meta::v1::Time, + jiff::{SignedDuration, Timestamp}, }, kube::{ ResourceExt, @@ -39,6 +41,12 @@ pub const TOPIC_FULL_CONTROLLER_NAME: &str = const AGENT_UNAVAILABLE_REASON: &str = "AgentUnavailable"; +/// `DeploymentConditionType` of `apps/v1`, true once the minimum number of replicas is available +const DEPLOYMENT_CONDITION_AVAILABLE: &str = "Available"; + +/// Shorter outages of the agent are not reported, such as a rollout that pulls a new image. +const AGENT_UNAVAILABLE_GRACE_PERIOD: SignedDuration = SignedDuration::from_secs(60); + #[derive(Debug, Snafu, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { @@ -88,9 +96,17 @@ enum ClusterState { ManagementAccessEnabled, } +#[derive(Debug, Eq, PartialEq)] +enum AgentState { + Available, + /// `None` if the agent Deployment does not exist + UnavailableSince(Option), +} + #[derive(Debug, Eq, PartialEq)] enum OperatorAction { LeaveToAgent, + WaitForAgent { remaining: std::time::Duration }, RemoveFinalizer, SetAgentUnavailableConditions { message: &'static str }, } @@ -122,11 +138,11 @@ pub async fn reconcile_topic( match operator_action_for( kafka_topic, &cluster_state_from(kafka_cluster.as_ref()), - agent_deployment - .as_ref() - .is_some_and(is_agent_deployment_available), + &agent_state_from(agent_deployment.as_ref()), + Timestamp::now(), ) { OperatorAction::LeaveToAgent => {} + OperatorAction::WaitForAgent { remaining } => return Ok(Action::requeue(remaining)), OperatorAction::RemoveFinalizer => remove_finalizer(kafka_topic, &ctx).await?, OperatorAction::SetAgentUnavailableConditions { message } => { set_agent_unavailable_conditions(kafka_topic, &ctx, message).await? @@ -188,7 +204,8 @@ fn cluster_state_from(kafka_cluster: Option<&DeserializeGuard>) -> fn operator_action_for( kafka_topic: &KafkaTopic, cluster_state: &ClusterState, - agent_available: bool, + agent_state: &AgentState, + now: Timestamp, ) -> OperatorAction { match cluster_state { ClusterState::MissingOrDeleting if kafka_topic.metadata.deletion_timestamp.is_some() => { @@ -209,23 +226,50 @@ fn operator_action_for( message: "the KafkaCluster has no agent, because clusterConfig.managementAccess is \ not enabled", }, - ClusterState::ManagementAccessEnabled if agent_available => OperatorAction::LeaveToAgent, - ClusterState::ManagementAccessEnabled => OperatorAction::SetAgentUnavailableConditions { - message: "the agent of the KafkaCluster is not available", + ClusterState::ManagementAccessEnabled => match agent_state { + AgentState::Available => OperatorAction::LeaveToAgent, + AgentState::UnavailableSince(Some(since)) + if now.duration_since(*since) < AGENT_UNAVAILABLE_GRACE_PERIOD => + { + OperatorAction::WaitForAgent { + remaining: (AGENT_UNAVAILABLE_GRACE_PERIOD - now.duration_since(*since)) + .unsigned_abs(), + } + } + AgentState::UnavailableSince(_) => OperatorAction::SetAgentUnavailableConditions { + message: "the agent of the KafkaCluster is not available", + }, }, } } -fn is_agent_deployment_available(agent_deployment: &Deployment) -> bool { - agent_deployment +/// A Deployment that has no `Available` condition yet is unavailable since its creation. +fn agent_state_from(agent_deployment: Option<&Deployment>) -> AgentState { + let Some(agent_deployment) = agent_deployment else { + return AgentState::UnavailableSince(None); + }; + let available_condition = agent_deployment .status .as_ref() .and_then(|status| status.conditions.as_ref()) - .is_some_and(|conditions| { + .and_then(|conditions| { conditions .iter() - .any(|condition| condition.type_ == "Available" && condition.status == "True") - }) + .find(|condition| condition.type_ == DEPLOYMENT_CONDITION_AVAILABLE) + }); + match available_condition { + Some(condition) if condition.status == "True" => AgentState::Available, + Some(condition) => { + AgentState::UnavailableSince(condition.last_transition_time.as_ref().map(|time| time.0)) + } + None => AgentState::UnavailableSince( + agent_deployment + .metadata + .creation_timestamp + .as_ref() + .map(|time| time.0), + ), + } } /// Only the conditions are applied, so the other fields stay owned by the agent and keep the @@ -331,43 +375,100 @@ mod tests { } } + fn now() -> Timestamp { + Timestamp::from_second(1_000).expect("the timestamp should be valid") + } + + fn seconds_ago(seconds: i64) -> Timestamp { + now() - SignedDuration::from_secs(seconds) + } + #[rstest] - #[case::cluster_missing(ClusterState::MissingOrDeleting, true, true)] - #[case::cluster_missing_without_agent(ClusterState::MissingOrDeleting, false, true)] - #[case::management_access_disabled(ClusterState::ManagementAccessDisabled, false, false)] - #[case::agent_available(ClusterState::ManagementAccessEnabled, true, false)] - #[case::agent_unavailable(ClusterState::ManagementAccessEnabled, false, false)] + #[case::cluster_missing(ClusterState::MissingOrDeleting, AgentState::Available, true)] + #[case::cluster_missing_without_agent( + ClusterState::MissingOrDeleting, + AgentState::UnavailableSince(None), + true + )] + #[case::management_access_disabled( + ClusterState::ManagementAccessDisabled, + AgentState::UnavailableSince(None), + false + )] + #[case::agent_available(ClusterState::ManagementAccessEnabled, AgentState::Available, false)] + #[case::agent_unavailable( + ClusterState::ManagementAccessEnabled, + AgentState::UnavailableSince(None), + false + )] fn remove_the_finalizer_only_if_the_cluster_is_missing_or_deleting( #[case] cluster_state: ClusterState, - #[case] agent_available: bool, + #[case] agent_state: AgentState, #[case] removes_finalizer: bool, ) { assert_eq!( removes_finalizer, - operator_action_for(&kafka_topic(true, true), &cluster_state, agent_available) - == OperatorAction::RemoveFinalizer + operator_action_for( + &kafka_topic(true, true), + &cluster_state, + &agent_state, + now() + ) == OperatorAction::RemoveFinalizer ); } #[rstest] - #[case::cluster_missing(ClusterState::MissingOrDeleting, true, true)] - #[case::management_access_disabled(ClusterState::ManagementAccessDisabled, true, true)] - #[case::agent_available(ClusterState::ManagementAccessEnabled, true, false)] - #[case::agent_unavailable(ClusterState::ManagementAccessEnabled, false, true)] + #[case::cluster_missing(ClusterState::MissingOrDeleting, AgentState::Available, true)] + #[case::management_access_disabled( + ClusterState::ManagementAccessDisabled, + AgentState::Available, + true + )] + #[case::agent_available(ClusterState::ManagementAccessEnabled, AgentState::Available, false)] + #[case::agent_deployment_missing( + ClusterState::ManagementAccessEnabled, + AgentState::UnavailableSince(None), + true + )] + #[case::agent_unavailable_for_longer_than_the_grace_period( + ClusterState::ManagementAccessEnabled, + AgentState::UnavailableSince(Some(seconds_ago(61))), + true + )] fn set_agent_unavailable_conditions_without_an_available_agent( #[case] cluster_state: ClusterState, - #[case] agent_available: bool, + #[case] agent_state: AgentState, #[case] sets_conditions: bool, ) { assert_eq!( sets_conditions, matches!( - operator_action_for(&kafka_topic(false, false), &cluster_state, agent_available), + operator_action_for( + &kafka_topic(false, false), + &cluster_state, + &agent_state, + now() + ), OperatorAction::SetAgentUnavailableConditions { .. } ) ); } + #[test] + fn wait_for_an_agent_that_is_unavailable_within_the_grace_period() { + assert_eq!( + OperatorAction::WaitForAgent { + remaining: std::time::Duration::from_secs(50) + }, + operator_action_for( + &kafka_topic(false, false), + &ClusterState::ManagementAccessEnabled, + &AgentState::UnavailableSince(Some(seconds_ago(10))), + now() + ) + ); + } + #[test] fn leave_a_deleting_kafka_topic_without_finalizer_alone() { assert_eq!( @@ -375,8 +476,41 @@ mod tests { operator_action_for( &kafka_topic(true, false), &ClusterState::MissingOrDeleting, - false + &AgentState::UnavailableSince(None), + now() ) ); } + + fn agent_deployment(available: Option<&str>) -> Deployment { + serde_json::from_value(json!({ + "metadata": { "creationTimestamp": "1970-01-01T00:10:00Z" }, + "status": { + "conditions": available.map(|status| vec![json!({ + "type": "Available", + "status": status, + "lastTransitionTime": "1970-01-01T00:15:00Z", + })]), + }, + })) + .expect("the Deployment should be valid") + } + + #[rstest] + #[case::missing(None, AgentState::UnavailableSince(None))] + #[case::available(Some(agent_deployment(Some("True"))), AgentState::Available)] + #[case::unavailable_since_the_transition( + Some(agent_deployment(Some("False"))), + AgentState::UnavailableSince(Some(Timestamp::from_second(900).expect("valid"))) + )] + #[case::unavailable_since_the_creation( + Some(agent_deployment(None)), + AgentState::UnavailableSince(Some(Timestamp::from_second(600).expect("valid"))) + )] + fn derive_the_agent_state_from_the_deployment( + #[case] agent_deployment: Option, + #[case] expected: AgentState, + ) { + assert_eq!(expected, agent_state_from(agent_deployment.as_ref())); + } } From 8b8934ce44fcd1e10140009793c4563ae3ddf35c Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 15:10:20 +0100 Subject: [PATCH 19/33] add docs and changelog entry --- CHANGELOG.md | 11 ++ docs/modules/kafka/pages/index.adoc | 2 + .../pages/usage-guide/management-access.adoc | 123 ++++++++++++++++ .../kafka/pages/usage-guide/topics.adoc | 135 ++++++++++++++++++ docs/modules/kafka/partials/nav.adoc | 3 + 5 files changed, 274 insertions(+) create mode 100644 docs/modules/kafka/pages/usage-guide/management-access.adoc create mode 100644 docs/modules/kafka/pages/usage-guide/topics.adoc diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fd40a4b..58d95c21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ All notable changes to this project will be documented in this file. - Support for Kafka 4.3.1 ([#1022]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#1029]). - Brokers now have a default affinity to the OPA Pods when OPA authorization is configured ([#1035]). +- Add management access via `spec.clusterConfig.managementAccess`. When enabled, the operator + deploys an agent per KafkaCluster, which authenticates anonymously or with a TLS client + certificate ([#1028]). +- Add the KafkaTopic CRD to manage topics declaratively. The agent creates and adopts topics, + replaces their configs with `configOverrides`, increases their partitions and optionally deletes + them. The operator removes the finalizers of KafkaTopics whose KafkaCluster is deleted ([#1050]). ### Changed @@ -53,6 +59,9 @@ All notable changes to this project will be documented in this file. - Support Vector 0.58.0 ([#1039], [#1044], [#1046]). - test: Update OPA to 1.20.2 in integration tests and docs example ([#1042]). - The operator Deployment runs with a restricted security context and a read-only root filesystem, so it can be deployed into namespaces that enforce the `restricted` Pod Security Standard ([#1049]). +- The `/ready` endpoint of the operator now also waits for the KafkaTopic CRD. If the CRDs are not + maintained by the operator (`maintenance.customResourceDefinitions.maintain: false`), install the + KafkaTopic CRD together with the KafkaCluster CRD ([#1050]). ### Fixed @@ -89,6 +98,7 @@ All notable changes to this project will be documented in this file. [#1021]: https://github.com/stackabletech/kafka-operator/pull/1021 [#1022]: https://github.com/stackabletech/kafka-operator/pull/1022 [#1024]: https://github.com/stackabletech/kafka-operator/pull/1024 +[#1028]: https://github.com/stackabletech/kafka-operator/pull/1028 [#1029]: https://github.com/stackabletech/kafka-operator/pull/1029 [#1034]: https://github.com/stackabletech/kafka-operator/pull/1034 [#1035]: https://github.com/stackabletech/kafka-operator/pull/1035 @@ -99,6 +109,7 @@ All notable changes to this project will be documented in this file. [#1046]: https://github.com/stackabletech/kafka-operator/pull/1046 [#1047]: https://github.com/stackabletech/kafka-operator/pull/1047 [#1049]: https://github.com/stackabletech/kafka-operator/pull/1049 +[#1050]: https://github.com/stackabletech/kafka-operator/pull/1050 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/kafka/pages/index.adoc b/docs/modules/kafka/pages/index.adoc index 13d1ac30..5b0c25a5 100644 --- a/docs/modules/kafka/pages/index.adoc +++ b/docs/modules/kafka/pages/index.adoc @@ -5,6 +5,7 @@ :github: https://github.com/stackabletech/kafka-operator/ :crd: {crd-ref}{crd-ref-release}&operator=kafka-operator :crd-kafkacluster: {crd-ref}/kafka.stackable.tech/v1alpha1/kafkacluster{crd-ref-release} +:crd-kafkatopic: {crd-ref}/kafka.stackable.tech/v1alpha1/kafkatopic{crd-ref-release} :feature-tracker: https://features.stackable.tech/unified :metadata-quorum: https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum :pegelonline: https://www.pegelonline.wsv.de/webservice/ueberblick @@ -75,3 +76,4 @@ include::partial$supported-versions.adoc[] * The {github}[kafka-operator {external-link-icon}^] GitHub repository * The operator feature overview in the {feature-tracker}[feature tracker {external-link-icon}^] * The {crd-kafkacluster}[KafkaCluster {external-link-icon}^] CRD documentation +* The {crd-kafkatopic}[KafkaTopic {external-link-icon}^] CRD documentation diff --git a/docs/modules/kafka/pages/usage-guide/management-access.adoc b/docs/modules/kafka/pages/usage-guide/management-access.adoc new file mode 100644 index 00000000..eebeb13e --- /dev/null +++ b/docs/modules/kafka/pages/usage-guide/management-access.adoc @@ -0,0 +1,123 @@ += Management access +:description: Let the Stackable Operator for Apache Kafka manage resources inside a Kafka cluster, such as topics, through a per-cluster agent. + +Management access allows the Stackable Data Platform (SDP) to manage resources inside an Apache Kafka cluster, such as xref:usage-guide/topics.adoc[topics]. + +== Agents + +The agent is an optional Deployment next to the Kafka cluster that manages resources inside Kafka, such as topics. +The operator deploys it when management access is enabled. + +== Limitations + +Management access is not available for all clusters: + +* Clusters that use ZooKeeper are not supported, only clusters in xref:usage-guide/kraft-controller.adoc[KRaft mode]. +* Clusters with Kerberos authentication are not supported. + +== Configuration + +Enable management access with `spec.clusterConfig.managementAccess` and choose how the agent authenticates to Kafka: + +[source,yaml] +---- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: simple-kafka +spec: + image: + productVersion: 4.3.1 + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls + authentication: + - authenticationClass: kafka-client-tls + managementAccess: + enabled: true # <1> + authentication: + userProvided: + tls: + secretClassName: kafka-client-auth-secret # <2> + controllers: + roleGroups: + default: + replicas: 3 + brokers: + roleGroups: + default: + replicas: 3 +---- +<1> Deploys the agent. Setting it to `false` removes the agent again. +<2> The SecretClass that provisions the client certificate of the agent, here the same SecretClass that the brokers use to verify client certificates, see xref:usage-guide/security.adoc[Security]. + +The agent supports the following authentication methods: + +`anonymous: {}`:: The agent connects without a client certificate. +This is only possible if the cluster does not require client authentication. +`tls.secretClassName`:: A client certificate provisioned by the xref:secret-operator:index.adoc[Secret Operator] from the given SecretClass. +`tls.secretName`:: A static Secret holding the client certificate in the keys `tls.crt` and `tls.key` (PEM), for example a Secret of type `kubernetes.io/tls`. + +The agent verifies the brokers with the CA of the SecretClass used by the client listener, that is `spec.clusterConfig.tls.serverSecretClass` or the `clientCertSecretClass` of the TLS AuthenticationClass. + +== Authorization + +The principal of the agent must be allowed to manage topics in Kafka, for example by xref:usage-guide/security.adoc#authorization[OPA authorization]. +The principal is derived from the subject of the client certificate. +The agent needs the following permissions: + +[cols="2,3", options="header"] +|=== +| Kafka API | Operation + +| Metadata | `DESCRIBE` on the topic +| CreateTopics | `CREATE` on the cluster or the topic, and `DESCRIBE_CONFIGS` on the topic +| CreatePartitions | `ALTER` on the topic +| DescribeConfigs | `DESCRIBE_CONFIGS` on the topic +| IncrementalAlterConfigs | `ALTER_CONFIGS` on the topic +| DeleteTopics | `DELETE` on the topic +| ListPartitionReassignments | `DESCRIBE` on the cluster +|=== + +NOTE: Without `DESCRIBE_CONFIGS`, Kafka still creates a topic, but the agent cannot observe it. +The KafkaTopic then reports `KafkaRejected`. + +== Network + +The agent connects to the headless Services of the broker role groups, which resolve to the broker Pods. +The traffic of the agent therefore stays within the Kubernetes cluster, regardless of the xref:usage-guide/listenerclass.adoc[ListenerClasses] of the cluster. + +== Agent resources + +The operator creates the following resources for the agent, named after the KafkaCluster: + +* A Deployment `-agent` with a single replica, limited to 500m CPU and 128Mi memory by default. +* A ServiceAccount `-agent`. +* A RoleBinding `-agent`, which binds the ClusterRole `kafka-agent-clusterrole` deployed by the Helm chart. + +The Deployment can be adjusted with xref:concepts:overrides.adoc#object-overrides[object overrides], for example to change the resources of the agent: + +[source,yaml] +---- +spec: + objectOverrides: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: simple-kafka-agent + spec: + template: + spec: + containers: + - name: agent + resources: + limits: + memory: 256Mi +---- + +== Disabling management access + +Setting `enabled` to `false` or removing `managementAccess` removes the agent. +The KafkaTopics and the topics in Kafka stay as they are. +While no agent is available, the operator reports `AgentUnavailable` in the conditions of the KafkaTopics of the cluster, see xref:usage-guide/topics.adoc#deletion[Deletion] for the effect on deleted KafkaTopics. diff --git a/docs/modules/kafka/pages/usage-guide/topics.adoc b/docs/modules/kafka/pages/usage-guide/topics.adoc new file mode 100644 index 00000000..67d27d87 --- /dev/null +++ b/docs/modules/kafka/pages/usage-guide/topics.adoc @@ -0,0 +1,135 @@ += Topics +:description: Manage Apache Kafka topics declaratively with KafkaTopic resources and the Stackable Operator for Apache Kafka. + +A KafkaTopic declares the desired state of a topic in an Apache Kafka cluster: its name, partitions, replication factor and configs. +The xref:usage-guide/management-access.adoc#_agents[agent] of the cluster creates the topic and keeps it in this state. +This requires xref:usage-guide/management-access.adoc[management access] to be enabled on the KafkaCluster. + +The KafkaTopic is the source of truth for its topic: changes made to the topic outside of the KafkaTopic, for example with `kafka-configs.sh`, are reverted. +The agent checks every topic at least every five minutes. + +NOTE: Other operators, such as Strimzi, also provide a resource of the kind `KafkaTopic`. +If both are installed, use the full resource name with `kubectl`, for example `kubectl get kafkatopics.kafka.stackable.tech`. + +== Configuration + +[source,yaml] +---- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterRef: + name: simple-kafka # <1> + topicName: shop.orders # <2> + partitions: 6 # <3> + replicationFactor: 3 # <4> + configOverrides: # <5> + retention.ms: "604800000" + cleanup.policy: delete + deletionPolicy: Retain # <6> +---- +<1> The KafkaCluster in the same namespace that manages the topic. It cannot be changed. +<2> The name of the topic in Kafka. Defaults to `metadata.name`. It cannot be changed. +<3> The number of partitions. Defaults to the broker setting `num.partitions` when the topic is created. +<4> The number of replicas of each partition. Defaults to the broker setting `default.replication.factor` when the topic is created. +<5> The configs of the topic, see <>. +<6> What happens to the topic when the KafkaTopic is deleted, see <>. + +=== Topic name + +A topic name consists of up to 249 characters out of `a-z`, `A-Z`, `0-9`, `.`, `_` and `-`, and cannot be `.` or `..`. +Kafka allows names that are not valid Kubernetes object names, for example with uppercase letters or underscores. +Such topics need `topicName`. + +Topics starting with `+__+` are internal to Kafka, such as `+__consumer_offsets+`, and are rejected. + +The topic name cannot be changed, and `topicName` can neither be added nor removed later. +To rename a topic, create a new KafkaTopic. + +=== Partitions and replication factor + +The number of partitions can only be increased, because Kafka cannot remove partitions. + +Changing the replication factor of an existing topic is not supported, because Kafka requires an explicit assignment of the replicas to brokers. +Use `kafka-reassign-partitions.sh` or Cruise Control for this. + +An unsupported change is reported in the `Synced` condition, and all other changes are still applied. + +[[configs]] +=== Configs + +`configOverrides` replaces all configs set on the topic. +Configs that are set on the topic but missing in `configOverrides` are reset to the broker defaults. + +The throttles of partition reassignments, `leader.replication.throttled.replicas` and `follower.replication.throttled.replicas`, are left untouched and cannot be set in `configOverrides`. + +== Existing topics + +A KafkaTopic for a topic that already exists in Kafka takes over the topic. + +WARNING: Taking over a topic is destructive. +The configs of the topic are replaced by `configOverrides`, which can delete data, for example with a shorter `retention.ms` or another `cleanup.policy`. +With `deletionPolicy: Delete`, deleting the KafkaTopic also deletes a topic that existed before the KafkaTopic. + +A KafkaTopic refers to its topic by name. +If the topic is deleted and created again outside of the KafkaTopic, the KafkaTopic takes over the new topic as well. + +If several KafkaTopics declare the same topic in the same cluster, the oldest KafkaTopic manages the topic, and the others report a `Conflict`. +When the oldest KafkaTopic is deleted, the next one takes over the topic within five minutes. + +== Status + +The status shows the topic in Kafka: + +[source,yaml] +---- +status: + conditions: ... + topicId: UXyU9i5ARn6W00ON2taeWA # <1> + topicName: shop.orders # <2> + partitions: 6 + replicationFactor: 3 # <3> + config: # <4> + cleanup.policy: delete + retention.ms: "604800000" +---- +<1> The topic ID as shown by `kafka-topics.sh`. +<2> The name of the topic in Kafka. +<3> The replication factor, excluding replicas that a running reassignment removes. It is not set if the partitions have different replication factors. +<4> The configs set on the topic, without broker and default values. + +The `Ready` condition shows whether the topic exists in Kafka. +The `Synced` condition shows whether the topic matches the KafkaTopic. +If it does not, the reason and message of the condition explain why, for example an unsupported change, a request rejected by Kafka or a paused reconciliation. + +== Pausing + +The reconciliation of a single KafkaTopic can be paused, independently of the xref:usage-guide/operations/cluster-operations.adoc[cluster operations] of the KafkaCluster: + +[source,yaml] +---- +spec: + topicOperation: + reconciliationPaused: true +---- + +While paused, the agent does not change the topic in Kafka, but still updates the status. + +[[deletion]] +== Deletion + +With the default `deletionPolicy: Retain`, deleting a KafkaTopic leaves the topic in Kafka. + +With `deletionPolicy: Delete`, the agent deletes the topic in Kafka when the KafkaTopic is deleted, unless another KafkaTopic declares the same topic. +The KafkaTopic gets a finalizer once the agent manages the topic, so the deletion waits until the topic is deleted. +If the reconciliation of the KafkaTopic is paused, the deletion waits until it is resumed. +If the brokers cannot be reached, for example while the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], switching the KafkaTopic to `deletionPolicy: Retain` releases the deletion. + +When the KafkaCluster or its namespace is deleted, the operator removes the finalizers of its KafkaTopics, so the deletion does not block. +The topics are not deleted in Kafka in this case. +If the KafkaCluster is created again with the same PersistentVolumeClaims, its topics still exist, but without KafkaTopics. + +If the KafkaCluster exists but has no agent, for example because management access is disabled, the deletion waits for the agent. +In this case, the finalizer `kafka.stackable.tech/kafkatopic` can be removed manually, which leaves the topic in Kafka. diff --git a/docs/modules/kafka/partials/nav.adoc b/docs/modules/kafka/partials/nav.adoc index 7849e974..07064ed4 100644 --- a/docs/modules/kafka/partials/nav.adoc +++ b/docs/modules/kafka/partials/nav.adoc @@ -6,6 +6,8 @@ ** xref:kafka:usage-guide/listenerclass.adoc[] ** xref:kafka:usage-guide/storage-resources.adoc[] ** xref:kafka:usage-guide/security.adoc[] +** xref:kafka:usage-guide/management-access.adoc[] +** xref:kafka:usage-guide/topics.adoc[] ** xref:kafka:usage-guide/monitoring.adoc[] ** xref:kafka:usage-guide/logging.adoc[] ** xref:kafka:usage-guide/overrides.adoc[] @@ -18,6 +20,7 @@ * xref:kafka:reference/index.adoc[] ** xref:kafka:reference/crds.adoc[] *** {crd-ref}/kafka.stackable.tech/v1alpha1/kafkacluster{crd-ref-release}[KafkaCluster {external-link-icon}^] +*** {crd-ref}/kafka.stackable.tech/v1alpha1/kafkatopic{crd-ref-release}[KafkaTopic {external-link-icon}^] ** xref:kafka:reference/discovery.adoc[] ** xref:kafka:reference/commandline-parameters.adoc[] ** xref:kafka:reference/environment-variables.adoc[] From 3fab70399f4a597ae278ea8985626fb03f38ce6d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 16:27:20 +0100 Subject: [PATCH 20/33] regenerate nix --- Cargo.nix | 20 ++++++++++---------- crate-hashes.json | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 8e0e6801..341aae4b 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5510,7 +5510,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "k8s_version"; authors = [ @@ -10489,7 +10489,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_agent"; authors = [ @@ -10532,7 +10532,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_certs"; authors = [ @@ -10934,7 +10934,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_operator"; authors = [ @@ -11139,7 +11139,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -11174,7 +11174,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_shared"; authors = [ @@ -11255,7 +11255,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_telemetry"; authors = [ @@ -11365,7 +11365,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_versioned"; authors = [ @@ -11411,7 +11411,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11475,7 +11475,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 405952ac..fd83bfb0 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715" } \ No newline at end of file From 0c7956bc1a1fb260c74eaa14bd929fbe3714e21e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 16:31:18 +0100 Subject: [PATCH 21/33] test correct namespace deletion --- tests/templates/kuttl/management-access/20-assert.yaml | 5 ++++- tests/templates/kuttl/management-access/20-create-topic.yaml | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/templates/kuttl/management-access/20-assert.yaml b/tests/templates/kuttl/management-access/20-assert.yaml index 7d87800a..c062e63a 100644 --- a/tests/templates/kuttl/management-access/20-assert.yaml +++ b/tests/templates/kuttl/management-access/20-assert.yaml @@ -1,5 +1,6 @@ --- -# The agent authenticates to Kafka and creates the topic. +# The agent authenticates to Kafka and creates the topic. The KafkaTopic keeps its finalizer until +# the end of the test, so the deletion of the namespace only finishes if the operator removes it. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 @@ -8,6 +9,8 @@ apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaTopic metadata: name: orders + finalizers: + - kafka.stackable.tech/kafkatopic status: conditions: - type: Ready diff --git a/tests/templates/kuttl/management-access/20-create-topic.yaml b/tests/templates/kuttl/management-access/20-create-topic.yaml index 8a57a0e5..43967315 100644 --- a/tests/templates/kuttl/management-access/20-create-topic.yaml +++ b/tests/templates/kuttl/management-access/20-create-topic.yaml @@ -9,3 +9,4 @@ spec: topicName: orders partitions: 3 replicationFactor: 1 + deletionPolicy: Delete From 0d1d0ac988e5e8790e72567ffecf2f6d284bc416 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 16:32:00 +0100 Subject: [PATCH 22/33] add OPA to management access test --- .../kuttl/management-access/07-assert.yaml | 7 ++++ .../management-access/07-install-opa.yaml.j2 | 34 +++++++++++++++++ .../kuttl/management-access/07_kafka.rego.j2 | 37 +++++++++++++++++++ .../kuttl/management-access/10_kafka.yaml.j2 | 4 ++ .../kuttl/management-access/30-assert.yaml | 25 +++++++++++++ .../30-create-forbidden-topic.yaml | 9 +++++ 6 files changed, 116 insertions(+) create mode 100644 tests/templates/kuttl/management-access/07-assert.yaml create mode 100644 tests/templates/kuttl/management-access/07-install-opa.yaml.j2 create mode 100644 tests/templates/kuttl/management-access/07_kafka.rego.j2 create mode 100644 tests/templates/kuttl/management-access/30-assert.yaml create mode 100644 tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml diff --git a/tests/templates/kuttl/management-access/07-assert.yaml b/tests/templates/kuttl/management-access/07-assert.yaml new file mode 100644 index 00000000..e1112367 --- /dev/null +++ b/tests/templates/kuttl/management-access/07-assert.yaml @@ -0,0 +1,7 @@ +--- +# OPA authorizes the brokers and the agent with the documented permissions. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: kubectl wait --namespace $NAMESPACE --for=condition=available opaclusters.opa.stackable.tech/test-opa --timeout 301s diff --git a/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 b/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 new file mode 100644 index 00000000..f235db18 --- /dev/null +++ b/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + envsubst '$NAMESPACE' < 07_kafka.rego | + kubectl create configmap test-rego-kafka --namespace $NAMESPACE --from-file=kafka.rego=/dev/stdin + - script: kubectl label configmap test-rego-kafka --namespace $NAMESPACE opa.stackable.tech/bundle=true +--- +apiVersion: opa.stackable.tech/v1alpha1 +kind: OpaCluster +metadata: + name: test-opa +spec: + image: +{% if test_scenario['values']['opa-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['opa-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['opa-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['opa-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + servers: + config: + logging: + containers: + opa: + console: + level: INFO + loggers: + decision: + level: INFO + roleGroups: + default: {} diff --git a/tests/templates/kuttl/management-access/07_kafka.rego.j2 b/tests/templates/kuttl/management-access/07_kafka.rego.j2 new file mode 100644 index 00000000..4121aa2e --- /dev/null +++ b/tests/templates/kuttl/management-access/07_kafka.rego.j2 @@ -0,0 +1,37 @@ +{% set authentication = test_scenario['values']['management-access-authentication'] %} +package authz + +broker_principal_pattern := `^DC=local,DC=cluster,DC=svc,DC=$NAMESPACE,DC=test-kafka-broker-default-headless,DC=test-kafka-broker-default-[0-9]+,CN=generated certificate for pod$` + +{% if authentication == 'anonymous' %} +agent_principal_pattern := `^ANONYMOUS$` +{% elif authentication == 'secret-class' %} +# The agent Pod has no subdomain, so its subject DN contains no Service. +agent_principal_pattern := `^DC=local,DC=cluster,DC=svc,DC=$NAMESPACE,DC=test-kafka-agent-[a-z0-9]+-[a-z0-9]+,CN=generated certificate for pod$` +{% else %} +agent_principal_pattern := `^CN=kafka-agent$` +{% endif %} + +default allow := false + +allow if { + input.requestContext.listenerName == "INTERNAL" + regex.match(broker_principal_pattern, input.requestContext.principal.name) +} + +allow if { + regex.match(agent_principal_pattern, input.requestContext.principal.name) + agent_action_is_allowed +} + +# The permissions of the agent as documented, limited to topics starting with "orders" +agent_action_is_allowed if { + input.action.resourcePattern.resourceType == "TOPIC" + startswith(input.action.resourcePattern.name, "orders") + input.action.operation in ["DESCRIBE", "CREATE", "ALTER", "DESCRIBE_CONFIGS", "ALTER_CONFIGS", "DELETE"] +} + +agent_action_is_allowed if { + input.action.resourcePattern.resourceType == "CLUSTER" + input.action.operation == "DESCRIBE" +} diff --git a/tests/templates/kuttl/management-access/10_kafka.yaml.j2 b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 index ca297698..58608e7b 100644 --- a/tests/templates/kuttl/management-access/10_kafka.yaml.j2 +++ b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 @@ -51,6 +51,10 @@ spec: authentication: - authenticationClass: client-auth-$NAMESPACE {% endif %} + authorization: + opa: + configMapName: test-opa + package: authz managementAccess: enabled: true authentication: diff --git a/tests/templates/kuttl/management-access/30-assert.yaml b/tests/templates/kuttl/management-access/30-assert.yaml new file mode 100644 index 00000000..56bcaf4d --- /dev/null +++ b/tests/templates/kuttl/management-access/30-assert.yaml @@ -0,0 +1,25 @@ +--- +# Kafka rejects the requests of the agent, and the KafkaTopic reports why. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: forbidden + ref: forbidden +assertAll: + - celExpr: "forbidden.status.conditions.exists(c, c.type == 'Synced' && c.message.contains('TopicAuthorizationFailed'))" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: forbidden +status: + conditions: + - type: Ready + status: Unknown + reason: KafkaRejected + - type: Synced + status: "False" + reason: KafkaRejected diff --git a/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml b/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml new file mode 100644 index 00000000..bc5e7332 --- /dev/null +++ b/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml @@ -0,0 +1,9 @@ +--- +# OPA does not allow the agent to manage this topic. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: forbidden +spec: + clusterRef: + name: test-kafka From f539947a436e4cfe3284916bf552ae7f090cbb0d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 18:10:55 +0100 Subject: [PATCH 23/33] remove OPA from docs and test --- .../pages/usage-guide/management-access.adoc | 2 +- .../kuttl/management-access/07-assert.yaml | 7 ---- .../management-access/07-install-opa.yaml.j2 | 34 ----------------- .../kuttl/management-access/07_kafka.rego.j2 | 37 ------------------- .../kuttl/management-access/10_kafka.yaml.j2 | 4 -- .../kuttl/management-access/30-assert.yaml | 25 ------------- .../30-create-forbidden-topic.yaml | 9 ----- 7 files changed, 1 insertion(+), 117 deletions(-) delete mode 100644 tests/templates/kuttl/management-access/07-assert.yaml delete mode 100644 tests/templates/kuttl/management-access/07-install-opa.yaml.j2 delete mode 100644 tests/templates/kuttl/management-access/07_kafka.rego.j2 delete mode 100644 tests/templates/kuttl/management-access/30-assert.yaml delete mode 100644 tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml diff --git a/docs/modules/kafka/pages/usage-guide/management-access.adoc b/docs/modules/kafka/pages/usage-guide/management-access.adoc index eebeb13e..650db4b5 100644 --- a/docs/modules/kafka/pages/usage-guide/management-access.adoc +++ b/docs/modules/kafka/pages/usage-guide/management-access.adoc @@ -63,7 +63,7 @@ The agent verifies the brokers with the CA of the SecretClass used by the client == Authorization -The principal of the agent must be allowed to manage topics in Kafka, for example by xref:usage-guide/security.adoc#authorization[OPA authorization]. +The principal of the agent must be allowed to manage topics in Kafka. The principal is derived from the subject of the client certificate. The agent needs the following permissions: diff --git a/tests/templates/kuttl/management-access/07-assert.yaml b/tests/templates/kuttl/management-access/07-assert.yaml deleted file mode 100644 index e1112367..00000000 --- a/tests/templates/kuttl/management-access/07-assert.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# OPA authorizes the brokers and the agent with the documented permissions. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -commands: - - script: kubectl wait --namespace $NAMESPACE --for=condition=available opaclusters.opa.stackable.tech/test-opa --timeout 301s diff --git a/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 b/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 deleted file mode 100644 index f235db18..00000000 --- a/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 +++ /dev/null @@ -1,34 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - envsubst '$NAMESPACE' < 07_kafka.rego | - kubectl create configmap test-rego-kafka --namespace $NAMESPACE --from-file=kafka.rego=/dev/stdin - - script: kubectl label configmap test-rego-kafka --namespace $NAMESPACE opa.stackable.tech/bundle=true ---- -apiVersion: opa.stackable.tech/v1alpha1 -kind: OpaCluster -metadata: - name: test-opa -spec: - image: -{% if test_scenario['values']['opa-latest'].find(",") > 0 %} - custom: "{{ test_scenario['values']['opa-latest'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['opa-latest'].split(',')[0] }}" -{% else %} - productVersion: "{{ test_scenario['values']['opa-latest'] }}" -{% endif %} - pullPolicy: IfNotPresent - servers: - config: - logging: - containers: - opa: - console: - level: INFO - loggers: - decision: - level: INFO - roleGroups: - default: {} diff --git a/tests/templates/kuttl/management-access/07_kafka.rego.j2 b/tests/templates/kuttl/management-access/07_kafka.rego.j2 deleted file mode 100644 index 4121aa2e..00000000 --- a/tests/templates/kuttl/management-access/07_kafka.rego.j2 +++ /dev/null @@ -1,37 +0,0 @@ -{% set authentication = test_scenario['values']['management-access-authentication'] %} -package authz - -broker_principal_pattern := `^DC=local,DC=cluster,DC=svc,DC=$NAMESPACE,DC=test-kafka-broker-default-headless,DC=test-kafka-broker-default-[0-9]+,CN=generated certificate for pod$` - -{% if authentication == 'anonymous' %} -agent_principal_pattern := `^ANONYMOUS$` -{% elif authentication == 'secret-class' %} -# The agent Pod has no subdomain, so its subject DN contains no Service. -agent_principal_pattern := `^DC=local,DC=cluster,DC=svc,DC=$NAMESPACE,DC=test-kafka-agent-[a-z0-9]+-[a-z0-9]+,CN=generated certificate for pod$` -{% else %} -agent_principal_pattern := `^CN=kafka-agent$` -{% endif %} - -default allow := false - -allow if { - input.requestContext.listenerName == "INTERNAL" - regex.match(broker_principal_pattern, input.requestContext.principal.name) -} - -allow if { - regex.match(agent_principal_pattern, input.requestContext.principal.name) - agent_action_is_allowed -} - -# The permissions of the agent as documented, limited to topics starting with "orders" -agent_action_is_allowed if { - input.action.resourcePattern.resourceType == "TOPIC" - startswith(input.action.resourcePattern.name, "orders") - input.action.operation in ["DESCRIBE", "CREATE", "ALTER", "DESCRIBE_CONFIGS", "ALTER_CONFIGS", "DELETE"] -} - -agent_action_is_allowed if { - input.action.resourcePattern.resourceType == "CLUSTER" - input.action.operation == "DESCRIBE" -} diff --git a/tests/templates/kuttl/management-access/10_kafka.yaml.j2 b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 index 58608e7b..ca297698 100644 --- a/tests/templates/kuttl/management-access/10_kafka.yaml.j2 +++ b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 @@ -51,10 +51,6 @@ spec: authentication: - authenticationClass: client-auth-$NAMESPACE {% endif %} - authorization: - opa: - configMapName: test-opa - package: authz managementAccess: enabled: true authentication: diff --git a/tests/templates/kuttl/management-access/30-assert.yaml b/tests/templates/kuttl/management-access/30-assert.yaml deleted file mode 100644 index 56bcaf4d..00000000 --- a/tests/templates/kuttl/management-access/30-assert.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Kafka rejects the requests of the agent, and the KafkaTopic reports why. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -resourceRefs: - - apiVersion: kafka.stackable.tech/v1alpha1 - kind: KafkaTopic - name: forbidden - ref: forbidden -assertAll: - - celExpr: "forbidden.status.conditions.exists(c, c.type == 'Synced' && c.message.contains('TopicAuthorizationFailed'))" ---- -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaTopic -metadata: - name: forbidden -status: - conditions: - - type: Ready - status: Unknown - reason: KafkaRejected - - type: Synced - status: "False" - reason: KafkaRejected diff --git a/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml b/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml deleted file mode 100644 index bc5e7332..00000000 --- a/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# OPA does not allow the agent to manage this topic. -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaTopic -metadata: - name: forbidden -spec: - clusterRef: - name: test-kafka From 0f390031254834736c248467f3320cbae7854fd7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 18:25:07 +0100 Subject: [PATCH 24/33] write status if failing during finalizer reconciliation --- .../kafka/pages/usage-guide/topics.adoc | 3 +- rust/agent-binary/src/topic/controller.rs | 31 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/docs/modules/kafka/pages/usage-guide/topics.adoc b/docs/modules/kafka/pages/usage-guide/topics.adoc index 67d27d87..1578137f 100644 --- a/docs/modules/kafka/pages/usage-guide/topics.adoc +++ b/docs/modules/kafka/pages/usage-guide/topics.adoc @@ -125,7 +125,8 @@ With the default `deletionPolicy: Retain`, deleting a KafkaTopic leaves the topi With `deletionPolicy: Delete`, the agent deletes the topic in Kafka when the KafkaTopic is deleted, unless another KafkaTopic declares the same topic. The KafkaTopic gets a finalizer once the agent manages the topic, so the deletion waits until the topic is deleted. If the reconciliation of the KafkaTopic is paused, the deletion waits until it is resumed. -If the brokers cannot be reached, for example while the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], switching the KafkaTopic to `deletionPolicy: Retain` releases the deletion. +If the brokers cannot be reached, for example while the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], the KafkaTopic reports `ConnectionFailed` and the agent retries the deletion. +Switching the KafkaTopic to `deletionPolicy: Retain` releases the deletion. When the KafkaCluster or its namespace is deleted, the operator removes the finalizers of its KafkaTopics, so the deletion does not block. The topics are not deleted in Kafka in this case. diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs index cdc9a496..1e7203b2 100644 --- a/rust/agent-binary/src/topic/controller.rs +++ b/rust/agent-binary/src/topic/controller.rs @@ -20,6 +20,7 @@ use tracing::info; use super::{ condition::TopicSyncOutcome, finalizer::{add_finalizer, reconcile_finalizer}, + plan::TopicState, sync::sync, update_status::update_status, }; @@ -103,8 +104,12 @@ pub async fn reconcile_topic( // Left untouched while paused, like operators that only update the status then. A deletion // waits until the reconciliation is resumed. let paused = kafka_topic.spec.topic_operation.reconciliation_paused; - if !paused && let Some(action) = reconcile_finalizer(kafka_topic, &ctx).await? { - return Ok(action); + if !paused { + match reconcile_finalizer(kafka_topic, &ctx).await { + Ok(Some(action)) => return Ok(action), + Ok(None) => {} + Err(error) => return fail_with_status(kafka_topic, &ctx, error, None).await, + } } // sync (Kafka) @@ -123,13 +128,23 @@ pub async fn reconcile_topic( update_status(kafka_topic, &ctx, &sync_outcome, topic_state.as_ref()).await?; Ok(requeue(&sync_outcome)) } - Err(Error::Kafka { source }) => { - let sync_outcome = TopicSyncOutcome::from_client_error(&source); - update_status(kafka_topic, &ctx, &sync_outcome, topic_state.as_ref()).await?; - Err(Error::Kafka { source }) - } - Err(error) => Err(error), + Err(error) => fail_with_status(kafka_topic, &ctx, error, topic_state.as_ref()).await, + } +} + +/// Reports a Kafka error in the status of the `KafkaTopic` before returning it. Other errors are +/// returned as they are. +async fn fail_with_status( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + error: Error, + topic_state: Option<&TopicState>, +) -> Result { + if let Error::Kafka { source } = &error { + let sync_outcome = TopicSyncOutcome::from_client_error(source); + update_status(kafka_topic, ctx, &sync_outcome, topic_state).await?; } + Err(error) } fn requeue(sync_outcome: &TopicSyncOutcome) -> Action { From d4cb53a88f42d583a831a2fdd436ccf634c1df05 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 18:33:30 +0100 Subject: [PATCH 25/33] improve docs on deletion --- docs/modules/kafka/pages/usage-guide/topics.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/kafka/pages/usage-guide/topics.adoc b/docs/modules/kafka/pages/usage-guide/topics.adoc index 1578137f..c060cdb9 100644 --- a/docs/modules/kafka/pages/usage-guide/topics.adoc +++ b/docs/modules/kafka/pages/usage-guide/topics.adoc @@ -125,12 +125,12 @@ With the default `deletionPolicy: Retain`, deleting a KafkaTopic leaves the topi With `deletionPolicy: Delete`, the agent deletes the topic in Kafka when the KafkaTopic is deleted, unless another KafkaTopic declares the same topic. The KafkaTopic gets a finalizer once the agent manages the topic, so the deletion waits until the topic is deleted. If the reconciliation of the KafkaTopic is paused, the deletion waits until it is resumed. -If the brokers cannot be reached, for example while the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], the KafkaTopic reports `ConnectionFailed` and the agent retries the deletion. +If the brokers cannot be reached, the KafkaTopic reports `ConnectionFailed` and the agent retries the deletion. Switching the KafkaTopic to `deletionPolicy: Retain` releases the deletion. When the KafkaCluster or its namespace is deleted, the operator removes the finalizers of its KafkaTopics, so the deletion does not block. The topics are not deleted in Kafka in this case. If the KafkaCluster is created again with the same PersistentVolumeClaims, its topics still exist, but without KafkaTopics. -If the KafkaCluster exists but has no agent, for example because management access is disabled, the deletion waits for the agent. +If the KafkaCluster exists but has no agent, for example because management access is disabled or the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], the deletion waits for the agent. In this case, the finalizer `kafka.stackable.tech/kafkatopic` can be removed manually, which leaves the topic in Kafka. From c0f9338964b28f5b872a93fd4b552484a90d9ee3 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 18:38:21 +0100 Subject: [PATCH 26/33] report a stopped Kafka cluster as unavailable --- rust/operator-binary/src/topic_controller.rs | 28 +++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/topic_controller.rs b/rust/operator-binary/src/topic_controller.rs index b7492c94..11aa540b 100644 --- a/rust/operator-binary/src/topic_controller.rs +++ b/rust/operator-binary/src/topic_controller.rs @@ -99,7 +99,7 @@ enum ClusterState { #[derive(Debug, Eq, PartialEq)] enum AgentState { Available, - /// `None` if the agent Deployment does not exist + /// `None` if the agent Deployment does not exist or has no replicas UnavailableSince(Option), } @@ -243,11 +243,21 @@ fn operator_action_for( } } -/// A Deployment that has no `Available` condition yet is unavailable since its creation. +/// A Deployment that has no `Available` condition yet is unavailable since its creation. One with +/// zero replicas, e.g. of a stopped KafkaCluster, is unavailable at once, although Kubernetes +/// reports it as `Available`. fn agent_state_from(agent_deployment: Option<&Deployment>) -> AgentState { let Some(agent_deployment) = agent_deployment else { return AgentState::UnavailableSince(None); }; + if agent_deployment + .spec + .as_ref() + .and_then(|spec| spec.replicas) + == Some(0) + { + return AgentState::UnavailableSince(None); + } let available_condition = agent_deployment .status .as_ref() @@ -353,7 +363,7 @@ async fn remove_finalizer(kafka_topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { #[cfg(test)] mod tests { use rstest::rstest; - use stackable_operator::kube::api::ObjectMeta; + use stackable_operator::{k8s_openapi::api::apps::v1::DeploymentSpec, kube::api::ObjectMeta}; use super::*; @@ -496,9 +506,21 @@ mod tests { .expect("the Deployment should be valid") } + fn scaled_to_zero(mut agent_deployment: Deployment) -> Deployment { + agent_deployment.spec = Some(DeploymentSpec { + replicas: Some(0), + ..DeploymentSpec::default() + }); + agent_deployment + } + #[rstest] #[case::missing(None, AgentState::UnavailableSince(None))] #[case::available(Some(agent_deployment(Some("True"))), AgentState::Available)] + #[case::scaled_to_zero( + Some(scaled_to_zero(agent_deployment(Some("True")))), + AgentState::UnavailableSince(None) + )] #[case::unavailable_since_the_transition( Some(agent_deployment(Some("False"))), AgentState::UnavailableSince(Some(Timestamp::from_second(900).expect("valid"))) From 08c6a41682f83ebb80100a9c1faf6d1ce7ecdcac Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 18:50:17 +0100 Subject: [PATCH 27/33] derive the topic configs from the applied changes --- rust/agent-binary/src/topic/plan.rs | 38 +++++++++++++++++++++++++++++ rust/agent-binary/src/topic/sync.rs | 18 ++++++-------- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs index ca70fca8..2fea70fd 100644 --- a/rust/agent-binary/src/topic/plan.rs +++ b/rust/agent-binary/src/topic/plan.rs @@ -49,6 +49,21 @@ impl TopicState { topic_configs: created.configs, } } + + /// The topic-level configs as they are after the changes are applied. + pub fn apply_config_changes( + &mut self, + configs_to_set: &BTreeMap, + configs_to_reset: &BTreeSet, + ) { + self.topic_configs + .retain(|key, _| !configs_to_reset.contains(key)); + self.topic_configs.extend( + configs_to_set + .iter() + .map(|(key, value)| (key.clone(), value.clone())), + ); + } } #[derive(Debug, Eq, PartialEq)] @@ -211,6 +226,8 @@ fn target_replication_factors(observed: &ObservedTopic) -> Vec { #[cfg(test)] mod tests { + use std::str::FromStr; + use stackable_kafka_admin::client::Partition; use super::*; @@ -379,6 +396,27 @@ mod tests { ); } + #[test] + fn apply_config_changes_to_the_topic_state() { + let mut topic_state = TopicState::observed( + TopicName::from_str("orders").expect("the topic name should be valid"), + &observed( + &[1], + &[("retention.ms", "1000"), ("cleanup.policy", "compact")], + ), + ); + + topic_state.apply_config_changes( + &configs(&[("retention.ms", "2000"), ("max.message.bytes", "10")]), + &keys(&["cleanup.policy"]), + ); + + assert_eq!( + configs(&[("max.message.bytes", "10"), ("retention.ms", "2000")]), + topic_state.topic_configs + ); + } + #[test] fn report_a_different_replication_factor_as_unsupported() { let observed = observed(&[3, 3], &[]); diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs index 97f98028..14bc9b65 100644 --- a/rust/agent-binary/src/topic/sync.rs +++ b/rust/agent-binary/src/topic/sync.rs @@ -136,7 +136,7 @@ async fn execute( // Config if !topic_update.configs_to_set.is_empty() || !topic_update.configs_to_reset.is_empty() { - let topic_configs = replace_configs( + alter_configs( kafka_topic, admin_client, &topic_name, @@ -145,7 +145,10 @@ async fn execute( ) .await?; if let Some(topic_state) = topic_state { - topic_state.topic_configs = topic_configs; + topic_state.apply_config_changes( + &topic_update.configs_to_set, + &topic_update.configs_to_reset, + ); } } } @@ -204,15 +207,13 @@ async fn grow_partitions( Ok(()) } -/// Returns the topic-level configs after the change, described again because -/// IncrementalAlterConfigs does not return them. -async fn replace_configs( +async fn alter_configs( kafka_topic: &KafkaTopic, admin_client: &mut KafkaAdminClient, topic_name: &TopicName, configs_to_set: &BTreeMap, configs_to_reset: &BTreeSet, -) -> Result> { +) -> Result<()> { admin_client .incremental_alter_configs(topic_name.as_ref(), configs_to_set, configs_to_reset) .await @@ -224,8 +225,5 @@ async fn replace_configs( kafka_topic = %kafka_topic.name_any(), "Changed the topic configs" ); - admin_client - .describe_topic_configs(topic_name.as_ref()) - .await - .context(KafkaSnafu) + Ok(()) } From b5be3c18bf6214cc56a855516d9b4c58c4d69500 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 21:04:07 +0100 Subject: [PATCH 28/33] add pull secret to docs --- docs/modules/kafka/pages/usage-guide/management-access.adoc | 4 ++++ docs/modules/kafka/pages/usage-guide/topics.adoc | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/modules/kafka/pages/usage-guide/management-access.adoc b/docs/modules/kafka/pages/usage-guide/management-access.adoc index 650db4b5..5373e797 100644 --- a/docs/modules/kafka/pages/usage-guide/management-access.adoc +++ b/docs/modules/kafka/pages/usage-guide/management-access.adoc @@ -8,6 +8,9 @@ Management access allows the Stackable Data Platform (SDP) to manage resources i The agent is an optional Deployment next to the Kafka cluster that manages resources inside Kafka, such as topics. The operator deploys it when management access is enabled. +The agent runs the image of the operator, pulled with the pull secrets of the Kafka image in `spec.image.pullSecrets`. +If the operator image is not public, these pull secrets must grant access to its registry as well, or the Deployment of the agent gets its own pull secrets with <>. + == Limitations Management access is not available for all clusters: @@ -88,6 +91,7 @@ The KafkaTopic then reports `KafkaRejected`. The agent connects to the headless Services of the broker role groups, which resolve to the broker Pods. The traffic of the agent therefore stays within the Kubernetes cluster, regardless of the xref:usage-guide/listenerclass.adoc[ListenerClasses] of the cluster. +[[agent-resources]] == Agent resources The operator creates the following resources for the agent, named after the KafkaCluster: diff --git a/docs/modules/kafka/pages/usage-guide/topics.adoc b/docs/modules/kafka/pages/usage-guide/topics.adoc index c060cdb9..c5501fd8 100644 --- a/docs/modules/kafka/pages/usage-guide/topics.adoc +++ b/docs/modules/kafka/pages/usage-guide/topics.adoc @@ -102,7 +102,7 @@ status: The `Ready` condition shows whether the topic exists in Kafka. The `Synced` condition shows whether the topic matches the KafkaTopic. -If it does not, the reason and message of the condition explain why, for example an unsupported change, a request rejected by Kafka or a paused reconciliation. +If it does not, the reason and message of the condition explain why, for example an unsupported change, a request rejected by Kafka, a paused reconciliation or an agent that is not running. == Pausing From 77eb95a86d75d4ffa68b1b9b4169199dad0e8ce5 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 21:05:46 +0100 Subject: [PATCH 29/33] remove the unreachable throttle filter from the planned config changes --- rust/agent-binary/src/topic/plan.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs index 2fea70fd..a204f137 100644 --- a/rust/agent-binary/src/topic/plan.rs +++ b/rust/agent-binary/src/topic/plan.rs @@ -164,7 +164,7 @@ fn unsupported_changes(spec: &KafkaTopicSpec, observed: &ObservedTopic) -> Vec, current_topic_configs: &BTreeMap, @@ -174,10 +174,7 @@ fn config_changes( // without drift does not write to Kafka. let configs_to_set = desired_topic_configs .iter() - .filter(|(key, value)| { - current_topic_configs.get(*key) != Some(value) - && !THROTTLE_CONFIG_KEYS.contains(&key.as_str()) - }) + .filter(|(key, value)| current_topic_configs.get(*key) != Some(value)) .map(|(key, value)| (key.clone(), value.clone())) .collect(); let configs_to_reset = current_topic_configs @@ -370,11 +367,7 @@ mod tests { &spec( None, None, - &[ - ("retention.ms", "2000"), - ("max.message.bytes", "10"), - ("follower.replication.throttled.replicas", "*"), - ], + &[("retention.ms", "2000"), ("max.message.bytes", "10")], ), Some(&observed), ); From af91710c7efb6965fd619400f2b7587df2d8292f Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 21:13:52 +0100 Subject: [PATCH 30/33] clarify the test-support feature of the shared crate --- rust/shared/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/shared/Cargo.toml b/rust/shared/Cargo.toml index 9eedd8c6..0e51994c 100644 --- a/rust/shared/Cargo.toml +++ b/rust/shared/Cargo.toml @@ -22,4 +22,5 @@ indoc.workspace = true rstest.workspace = true [features] +# Referenced by the attributed_string_type! macro of stackable-operator test-support = [] From 45b8a8cb6f6439c2624cbeb76a7cc29feb86722e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 21:21:38 +0100 Subject: [PATCH 31/33] document that the admin client serializes the Kafka requests --- rust/agent-binary/src/topic/controller.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs index 1e7203b2..e9b0f7f6 100644 --- a/rust/agent-binary/src/topic/controller.rs +++ b/rust/agent-binary/src/topic/controller.rs @@ -67,7 +67,8 @@ pub struct Ctx { pub(crate) client: stackable_operator::client::Client, pub(crate) kafka_topics: Store>, /// Reads the TLS files on every new connection and reconnects after a failed request, so - /// rotated certificates are picked up without restarting the agent. + /// rotated certificates are picked up without restarting the agent. The Kafka requests of + /// all reconciles are serialized on its single connection. pub(crate) admin_client: Mutex, } From c475bdb854585e5ae3171909aed20b837ba1c45c Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 30 Oct 2026 10:30:17 +0100 Subject: [PATCH 32/33] trim kuttl tests --- .../management-access/20-create-topic.yaml | 2 -- .../kuttl/topic/20-create-topic.yaml | 1 + .../kuttl/topic/30-change-topic.yaml | 5 --- .../topic/41-remove-config-override.yaml | 15 ++++---- .../kuttl/topic/50-unsupported-change.yaml | 5 --- tests/templates/kuttl/topic/51-assert.yaml | 20 +++++++++++ .../topic/51-revert-unsupported-change.yaml | 8 +++++ .../kuttl/topic/60-invalid-spec.yaml | 3 -- .../kuttl/topic/71-adopt-foreign-topic.yaml | 3 -- .../templates/kuttl/topic/75-pause-topic.yaml | 5 --- .../kuttl/topic/76-resume-topic.yaml | 7 ---- .../kuttl/topic/77-invalid-config-value.yaml | 2 -- .../topic/80-create-conflicting-topic.yaml | 2 -- .../kuttl/topic/92-create-cleanup-topic.yaml | 2 -- .../topic/95-disable-management-access.yaml | 9 +++++ .../95-disable-management-access.yaml.j2 | 35 ------------------- .../topic/96-delete-topic-without-agent.yaml | 2 +- 17 files changed, 47 insertions(+), 79 deletions(-) create mode 100644 tests/templates/kuttl/topic/51-assert.yaml create mode 100644 tests/templates/kuttl/topic/51-revert-unsupported-change.yaml create mode 100644 tests/templates/kuttl/topic/95-disable-management-access.yaml delete mode 100644 tests/templates/kuttl/topic/95-disable-management-access.yaml.j2 diff --git a/tests/templates/kuttl/management-access/20-create-topic.yaml b/tests/templates/kuttl/management-access/20-create-topic.yaml index 43967315..9da42728 100644 --- a/tests/templates/kuttl/management-access/20-create-topic.yaml +++ b/tests/templates/kuttl/management-access/20-create-topic.yaml @@ -6,7 +6,5 @@ metadata: spec: clusterRef: name: test-kafka - topicName: orders partitions: 3 - replicationFactor: 1 deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/20-create-topic.yaml b/tests/templates/kuttl/topic/20-create-topic.yaml index d64165b4..2cba7e02 100644 --- a/tests/templates/kuttl/topic/20-create-topic.yaml +++ b/tests/templates/kuttl/topic/20-create-topic.yaml @@ -6,6 +6,7 @@ metadata: spec: clusterRef: name: test-kafka + # Equal to the name on purpose, so that step 31 can test that topicName is immutable. topicName: orders partitions: 3 replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/30-change-topic.yaml b/tests/templates/kuttl/topic/30-change-topic.yaml index ef186c7b..911cc384 100644 --- a/tests/templates/kuttl/topic/30-change-topic.yaml +++ b/tests/templates/kuttl/topic/30-change-topic.yaml @@ -4,11 +4,6 @@ kind: KafkaTopic metadata: name: orders spec: - clusterRef: - name: test-kafka - topicName: orders partitions: 5 - replicationFactor: 1 configOverrides: retention.ms: "7200000" - max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/41-remove-config-override.yaml b/tests/templates/kuttl/topic/41-remove-config-override.yaml index ca178649..31e7866f 100644 --- a/tests/templates/kuttl/topic/41-remove-config-override.yaml +++ b/tests/templates/kuttl/topic/41-remove-config-override.yaml @@ -1,8 +1,9 @@ --- -# kuttl merges applied objects into the existing ones, so a removed key has to be removed explicitly. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: >- - kubectl patch kafkatopic orders --namespace $NAMESPACE --type json - --patch '[{"op": "remove", "path": "/spec/configOverrides/max.message.bytes"}]' +# kuttl merges applied objects into the existing ones, so a removed key has to be set to null. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + configOverrides: + max.message.bytes: null diff --git a/tests/templates/kuttl/topic/50-unsupported-change.yaml b/tests/templates/kuttl/topic/50-unsupported-change.yaml index 9424a96d..9831ad24 100644 --- a/tests/templates/kuttl/topic/50-unsupported-change.yaml +++ b/tests/templates/kuttl/topic/50-unsupported-change.yaml @@ -5,10 +5,5 @@ kind: KafkaTopic metadata: name: orders spec: - clusterRef: - name: test-kafka - topicName: orders partitions: 2 replicationFactor: 3 - configOverrides: - retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/51-assert.yaml b/tests/templates/kuttl/topic/51-assert.yaml new file mode 100644 index 00000000..2c43c156 --- /dev/null +++ b/tests/templates/kuttl/topic/51-assert.yaml @@ -0,0 +1,20 @@ +--- +# The topic is in sync again once the spec matches it. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml b/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml new file mode 100644 index 00000000..cfaeab1f --- /dev/null +++ b/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/60-invalid-spec.yaml b/tests/templates/kuttl/topic/60-invalid-spec.yaml index e331d366..85c919ce 100644 --- a/tests/templates/kuttl/topic/60-invalid-spec.yaml +++ b/tests/templates/kuttl/topic/60-invalid-spec.yaml @@ -7,8 +7,5 @@ metadata: spec: clusterRef: name: test-kafka - topicName: throttled - partitions: 1 - replicationFactor: 1 configOverrides: leader.replication.throttled.replicas: "*" diff --git a/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml b/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml index 71b6bedf..10395380 100644 --- a/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml +++ b/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml @@ -6,7 +6,4 @@ metadata: spec: clusterRef: name: test-kafka - topicName: legacy - partitions: 1 - replicationFactor: 1 deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/75-pause-topic.yaml b/tests/templates/kuttl/topic/75-pause-topic.yaml index ea35cd52..f66a8864 100644 --- a/tests/templates/kuttl/topic/75-pause-topic.yaml +++ b/tests/templates/kuttl/topic/75-pause-topic.yaml @@ -5,11 +5,6 @@ kind: KafkaTopic metadata: name: orders spec: - clusterRef: - name: test-kafka - topicName: orders - partitions: 5 - replicationFactor: 1 configOverrides: retention.ms: "10800000" topicOperation: diff --git a/tests/templates/kuttl/topic/76-resume-topic.yaml b/tests/templates/kuttl/topic/76-resume-topic.yaml index 8cc896dc..f2050561 100644 --- a/tests/templates/kuttl/topic/76-resume-topic.yaml +++ b/tests/templates/kuttl/topic/76-resume-topic.yaml @@ -4,12 +4,5 @@ kind: KafkaTopic metadata: name: orders spec: - clusterRef: - name: test-kafka - topicName: orders - partitions: 5 - replicationFactor: 1 - configOverrides: - retention.ms: "10800000" topicOperation: reconciliationPaused: false diff --git a/tests/templates/kuttl/topic/77-invalid-config-value.yaml b/tests/templates/kuttl/topic/77-invalid-config-value.yaml index a230cce9..f23d4445 100644 --- a/tests/templates/kuttl/topic/77-invalid-config-value.yaml +++ b/tests/templates/kuttl/topic/77-invalid-config-value.yaml @@ -5,7 +5,5 @@ kind: KafkaTopic metadata: name: defaults spec: - clusterRef: - name: test-kafka configOverrides: retention.ms: "abc" diff --git a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml index 7314dcc8..b213bb82 100644 --- a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml +++ b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml @@ -8,6 +8,4 @@ spec: clusterRef: name: test-kafka topicName: orders - partitions: 5 - replicationFactor: 1 deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml b/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml index f14412ee..6be86881 100644 --- a/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml +++ b/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml @@ -8,6 +8,4 @@ metadata: spec: clusterRef: name: test-kafka - partitions: 1 - replicationFactor: 1 deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/95-disable-management-access.yaml b/tests/templates/kuttl/topic/95-disable-management-access.yaml new file mode 100644 index 00000000..a48f7063 --- /dev/null +++ b/tests/templates/kuttl/topic/95-disable-management-access.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + clusterConfig: + managementAccess: + enabled: false diff --git a/tests/templates/kuttl/topic/95-disable-management-access.yaml.j2 b/tests/templates/kuttl/topic/95-disable-management-access.yaml.j2 deleted file mode 100644 index 10563bfa..00000000 --- a/tests/templates/kuttl/topic/95-disable-management-access.yaml.j2 +++ /dev/null @@ -1,35 +0,0 @@ ---- -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaCluster -metadata: - name: test-kafka -spec: - image: -{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} - custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" -{% else %} - productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" -{% endif %} - pullPolicy: IfNotPresent - clusterConfig: - metadataManager: KRaft - tls: - serverSecretClass: tls - managementAccess: - enabled: false - authentication: - userProvided: - anonymous: {} - controllers: - config: - gracefulShutdownTimeout: 30s # speed up tests - roleGroups: - default: - replicas: 1 - brokers: - config: - gracefulShutdownTimeout: 30s # speed up tests - roleGroups: - default: - replicas: 1 diff --git a/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml b/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml index 57614fa4..4962acaa 100644 --- a/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml +++ b/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml @@ -1,5 +1,5 @@ --- -# kuttl would wait for the deletion, which only finishes with the KafkaCluster. +# Not a kuttl delete, as that waits for the deletion, which only finishes with the KafkaCluster. apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: From 45d37f3190b60c925e14db713e9fe92b5f9a2ddd Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 30 Oct 2026 11:07:39 +0100 Subject: [PATCH 33/33] add keep environment util to topic test --- .../kuttl/topic/98-keep-environment.yaml | 7 ++++++ tests/templates/kuttl/topic/98_topic.yaml | 9 ++++++++ .../templates/kuttl/topic/keep-environment.sh | 23 +++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 tests/templates/kuttl/topic/98-keep-environment.yaml create mode 100644 tests/templates/kuttl/topic/98_topic.yaml create mode 100644 tests/templates/kuttl/topic/keep-environment.sh diff --git a/tests/templates/kuttl/topic/98-keep-environment.yaml b/tests/templates/kuttl/topic/98-keep-environment.yaml new file mode 100644 index 00000000..910bec3e --- /dev/null +++ b/tests/templates/kuttl/topic/98-keep-environment.yaml @@ -0,0 +1,7 @@ +--- +# With --skip-delete, the namespace is kept, so the deleted cluster is created again to experiment +# with. See the script for the details. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: bash keep-environment.sh diff --git a/tests/templates/kuttl/topic/98_topic.yaml b/tests/templates/kuttl/topic/98_topic.yaml new file mode 100644 index 00000000..d8aa89ea --- /dev/null +++ b/tests/templates/kuttl/topic/98_topic.yaml @@ -0,0 +1,9 @@ +--- +# Adopts the topic that the KafkaTopic orders retained in Kafka. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterRef: + name: test-kafka diff --git a/tests/templates/kuttl/topic/keep-environment.sh b/tests/templates/kuttl/topic/keep-environment.sh new file mode 100644 index 00000000..fcd274f9 --- /dev/null +++ b/tests/templates/kuttl/topic/keep-environment.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Creates the KafkaCluster and a KafkaTopic again after the deletion steps, if kuttl was started +# with --skip-delete, so that the kept namespace has a running cluster to experiment with. kuttl +# does not expose the flag to test steps, hence looking for it in the argv of the kuttl ancestor. +set -euo pipefail + +pid=$PPID +while [ "${pid:-0}" -gt 1 ]; do + args=$(ps -o args= -p "$pid" 2>/dev/null || true) + case "$args" in + *kuttl*--skip-delete*) break ;; + esac + pid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d '[:space:]' || true) +done +if [ "${pid:-0}" -le 1 ]; then + echo "kuttl deletes the namespace, nothing to keep" + exit 0 +fi + +# The cluster id is the name of the KafkaCluster, so the topics in the kept PersistentVolumeClaims +# are back once the cluster runs. +kubectl apply --namespace "$NAMESPACE" --filename 10-install-kafka.yaml +kubectl apply --namespace "$NAMESPACE" --filename 98_topic.yaml