diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b5817e55..e53edefeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ All notable changes to this project will be documented in this file. - Support for Kafka 4.3.1 ([#1022]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#1029]). - Brokers now have a default affinity to the OPA Pods when OPA authorization is configured ([#1035]). +- Add management access via `spec.clusterConfig.managementAccess`. When enabled, the operator + deploys an agent per KafkaCluster, which authenticates anonymously or with a TLS client + certificate ([#1028]). +- Add the KafkaTopic CRD to manage topics declaratively. The agent creates and adopts topics, + replaces their configs with `configOverrides`, increases their partitions and optionally deletes + them. The operator removes the finalizers of KafkaTopics whose KafkaCluster is deleted ([#1050]). ### Changed @@ -56,6 +62,9 @@ All notable changes to this project will be documented in this file. - Support Vector 0.58.0 ([#1039], [#1044], [#1046]). - test: Update OPA to 1.20.2 in integration tests and docs example ([#1042]). - The operator Deployment runs with a restricted security context and a read-only root filesystem, so it can be deployed into namespaces that enforce the `restricted` Pod Security Standard ([#1049]). +- The `/ready` endpoint of the operator now also waits for the KafkaTopic CRD. If the CRDs are not + maintained by the operator (`maintenance.customResourceDefinitions.maintain: false`), install the + KafkaTopic CRD together with the KafkaCluster CRD ([#1050]). ### Fixed @@ -92,6 +101,7 @@ All notable changes to this project will be documented in this file. [#1021]: https://github.com/stackabletech/kafka-operator/pull/1021 [#1022]: https://github.com/stackabletech/kafka-operator/pull/1022 [#1024]: https://github.com/stackabletech/kafka-operator/pull/1024 +[#1028]: https://github.com/stackabletech/kafka-operator/pull/1028 [#1029]: https://github.com/stackabletech/kafka-operator/pull/1029 [#1034]: https://github.com/stackabletech/kafka-operator/pull/1034 [#1035]: https://github.com/stackabletech/kafka-operator/pull/1035 @@ -103,6 +113,7 @@ All notable changes to this project will be documented in this file. [#1046]: https://github.com/stackabletech/kafka-operator/pull/1046 [#1047]: https://github.com/stackabletech/kafka-operator/pull/1047 [#1049]: https://github.com/stackabletech/kafka-operator/pull/1049 +[#1050]: https://github.com/stackabletech/kafka-operator/pull/1050 ## [26.7.0] - 2026-07-21 diff --git a/Cargo.lock b/Cargo.lock index a8f3cabea..9bd492205 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3240,7 +3240,15 @@ dependencies = [ "anyhow", "built", "clap", + "const_format", + "futures", + "rstest", + "serde_json", + "snafu 0.9.2", + "stackable-kafka-admin", + "stackable-kafka-shared", "stackable-operator", + "strum", "tokio", "tracing", ] @@ -3271,7 +3279,12 @@ dependencies = [ name = "stackable-kafka-shared" version = "0.0.0-dev" dependencies = [ + "indoc", + "regex", "rstest", + "serde", + "serde_json", + "snafu 0.9.2", "stackable-operator", "strum", ] diff --git a/Cargo.nix b/Cargo.nix index f76986f15..c70bdba96 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10646,11 +10646,40 @@ rec { name = "clap"; packageId = "clap"; } + { + name = "const_format"; + packageId = "const_format"; + } + { + name = "futures"; + packageId = "futures"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "snafu"; + packageId = "snafu 0.9.2"; + } + { + name = "stackable-kafka-admin"; + packageId = "stackable-kafka-admin"; + } + { + name = "stackable-kafka-shared"; + packageId = "stackable-kafka-shared"; + } { name = "stackable-operator"; packageId = "stackable-operator"; features = [ "agent" "crds" "webhook" ]; } + { + name = "strum"; + packageId = "strum"; + features = [ "derive" ]; + } { name = "tokio"; packageId = "tokio"; @@ -10668,6 +10697,12 @@ rec { features = [ "chrono" "git2" ]; } ]; + devDependencies = [ + { + name = "rstest"; + packageId = "rstest"; + } + ]; }; "stackable-kafka-operator" = rec { @@ -10772,6 +10807,23 @@ rec { "Stackable GmbH " ]; dependencies = [ + { + name = "regex"; + packageId = "regex"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "snafu"; + packageId = "snafu 0.9.2"; + } { name = "stackable-operator"; packageId = "stackable-operator"; @@ -10784,12 +10836,18 @@ rec { } ]; devDependencies = [ + { + name = "indoc"; + packageId = "indoc"; + } { name = "rstest"; packageId = "rstest"; } ]; - + features = { + }; + resolvedDefaultFeatures = [ "test-support" ]; }; "stackable-operator" = rec { crateName = "stackable-operator"; diff --git a/Cargo.toml b/Cargo.toml index 382a91b17..fffe1d0e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["agent", "crds", "webhook"] } +stackable-kafka-admin = { path = "rust/kafka-admin" } stackable-kafka-shared = { path = "rust/shared" } anyhow = "1.0" @@ -22,6 +23,7 @@ const_format = "0.2" futures = "0.3" indoc = "2.0" kafka-protocol = { version = "0.18", default-features = false, features = ["broker", "client"] } +regex = "1.13" rstest = "0.26" rustls = { version = "0.23", default-features = false, features = ["logging", "ring", "std", "tls12"] } semver = "1.0" diff --git a/deploy/helm/kafka-operator/Chart.yaml b/deploy/helm/kafka-operator/Chart.yaml index b37d3df18..6e7a2ceb2 100644 --- a/deploy/helm/kafka-operator/Chart.yaml +++ b/deploy/helm/kafka-operator/Chart.yaml @@ -42,3 +42,8 @@ annotations: name: kafkaclusters.kafka.stackable.tech displayName: KafkaCluster description: A Kafka cluster stacklet. This resource is managed by the Stackable operator for Apache Kafka. + - kind: KafkaTopic + version: v1alpha1 + name: kafkatopics.kafka.stackable.tech + displayName: KafkaTopic + description: A Kafka topic, created and updated by the agent of the referenced Kafka cluster. diff --git a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml index fb3f2995e..0de7e9b8c 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml @@ -8,6 +8,30 @@ metadata: labels: {{- include "kafka-operator.labels" . | nindent 4 }} rules: + # KafkaTopics of the agent's cluster: watched, and their finalizer added and removed. + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics + verbs: + - get + - list + - watch + - patch + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics/status + verbs: + - patch + # Required to report reconciliation errors as events on the KafkaTopic objects. + - apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - patch {{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} # On OpenShift, the agent pod must be allowed to use the nonroot-v2 SCC - apiGroups: diff --git a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml index aafed503c..f8f8193a1 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml @@ -128,6 +128,24 @@ rules: - {{ include "kafka-operator.name" . }}clusters/status verbs: - patch + # KafkaTopics: watched to take over while no agent is available, and their finalizer removed + # once the KafkaCluster is gone. + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics + verbs: + - get + - list + - watch + - patch + # Status subresource: conditions set while the agent is unavailable. + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics/status + verbs: + - patch # Read authentication class configuration referenced in the KafkaCluster spec. - apiGroups: - authentication.stackable.tech diff --git a/docs/modules/kafka/pages/index.adoc b/docs/modules/kafka/pages/index.adoc index 13d1ac30f..5b0c25a5c 100644 --- a/docs/modules/kafka/pages/index.adoc +++ b/docs/modules/kafka/pages/index.adoc @@ -5,6 +5,7 @@ :github: https://github.com/stackabletech/kafka-operator/ :crd: {crd-ref}{crd-ref-release}&operator=kafka-operator :crd-kafkacluster: {crd-ref}/kafka.stackable.tech/v1alpha1/kafkacluster{crd-ref-release} +:crd-kafkatopic: {crd-ref}/kafka.stackable.tech/v1alpha1/kafkatopic{crd-ref-release} :feature-tracker: https://features.stackable.tech/unified :metadata-quorum: https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum :pegelonline: https://www.pegelonline.wsv.de/webservice/ueberblick @@ -75,3 +76,4 @@ include::partial$supported-versions.adoc[] * The {github}[kafka-operator {external-link-icon}^] GitHub repository * The operator feature overview in the {feature-tracker}[feature tracker {external-link-icon}^] * The {crd-kafkacluster}[KafkaCluster {external-link-icon}^] CRD documentation +* The {crd-kafkatopic}[KafkaTopic {external-link-icon}^] CRD documentation diff --git a/docs/modules/kafka/pages/usage-guide/management-access.adoc b/docs/modules/kafka/pages/usage-guide/management-access.adoc new file mode 100644 index 000000000..5373e7976 --- /dev/null +++ b/docs/modules/kafka/pages/usage-guide/management-access.adoc @@ -0,0 +1,127 @@ += Management access +:description: Let the Stackable Operator for Apache Kafka manage resources inside a Kafka cluster, such as topics, through a per-cluster agent. + +Management access allows the Stackable Data Platform (SDP) to manage resources inside an Apache Kafka cluster, such as xref:usage-guide/topics.adoc[topics]. + +== Agents + +The agent is an optional Deployment next to the Kafka cluster that manages resources inside Kafka, such as topics. +The operator deploys it when management access is enabled. + +The agent runs the image of the operator, pulled with the pull secrets of the Kafka image in `spec.image.pullSecrets`. +If the operator image is not public, these pull secrets must grant access to its registry as well, or the Deployment of the agent gets its own pull secrets with <>. + +== Limitations + +Management access is not available for all clusters: + +* Clusters that use ZooKeeper are not supported, only clusters in xref:usage-guide/kraft-controller.adoc[KRaft mode]. +* Clusters with Kerberos authentication are not supported. + +== Configuration + +Enable management access with `spec.clusterConfig.managementAccess` and choose how the agent authenticates to Kafka: + +[source,yaml] +---- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: simple-kafka +spec: + image: + productVersion: 4.3.1 + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls + authentication: + - authenticationClass: kafka-client-tls + managementAccess: + enabled: true # <1> + authentication: + userProvided: + tls: + secretClassName: kafka-client-auth-secret # <2> + controllers: + roleGroups: + default: + replicas: 3 + brokers: + roleGroups: + default: + replicas: 3 +---- +<1> Deploys the agent. Setting it to `false` removes the agent again. +<2> The SecretClass that provisions the client certificate of the agent, here the same SecretClass that the brokers use to verify client certificates, see xref:usage-guide/security.adoc[Security]. + +The agent supports the following authentication methods: + +`anonymous: {}`:: The agent connects without a client certificate. +This is only possible if the cluster does not require client authentication. +`tls.secretClassName`:: A client certificate provisioned by the xref:secret-operator:index.adoc[Secret Operator] from the given SecretClass. +`tls.secretName`:: A static Secret holding the client certificate in the keys `tls.crt` and `tls.key` (PEM), for example a Secret of type `kubernetes.io/tls`. + +The agent verifies the brokers with the CA of the SecretClass used by the client listener, that is `spec.clusterConfig.tls.serverSecretClass` or the `clientCertSecretClass` of the TLS AuthenticationClass. + +== Authorization + +The principal of the agent must be allowed to manage topics in Kafka. +The principal is derived from the subject of the client certificate. +The agent needs the following permissions: + +[cols="2,3", options="header"] +|=== +| Kafka API | Operation + +| Metadata | `DESCRIBE` on the topic +| CreateTopics | `CREATE` on the cluster or the topic, and `DESCRIBE_CONFIGS` on the topic +| CreatePartitions | `ALTER` on the topic +| DescribeConfigs | `DESCRIBE_CONFIGS` on the topic +| IncrementalAlterConfigs | `ALTER_CONFIGS` on the topic +| DeleteTopics | `DELETE` on the topic +| ListPartitionReassignments | `DESCRIBE` on the cluster +|=== + +NOTE: Without `DESCRIBE_CONFIGS`, Kafka still creates a topic, but the agent cannot observe it. +The KafkaTopic then reports `KafkaRejected`. + +== Network + +The agent connects to the headless Services of the broker role groups, which resolve to the broker Pods. +The traffic of the agent therefore stays within the Kubernetes cluster, regardless of the xref:usage-guide/listenerclass.adoc[ListenerClasses] of the cluster. + +[[agent-resources]] +== Agent resources + +The operator creates the following resources for the agent, named after the KafkaCluster: + +* A Deployment `-agent` with a single replica, limited to 500m CPU and 128Mi memory by default. +* A ServiceAccount `-agent`. +* A RoleBinding `-agent`, which binds the ClusterRole `kafka-agent-clusterrole` deployed by the Helm chart. + +The Deployment can be adjusted with xref:concepts:overrides.adoc#object-overrides[object overrides], for example to change the resources of the agent: + +[source,yaml] +---- +spec: + objectOverrides: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: simple-kafka-agent + spec: + template: + spec: + containers: + - name: agent + resources: + limits: + memory: 256Mi +---- + +== Disabling management access + +Setting `enabled` to `false` or removing `managementAccess` removes the agent. +The KafkaTopics and the topics in Kafka stay as they are. +While no agent is available, the operator reports `AgentUnavailable` in the conditions of the KafkaTopics of the cluster, see xref:usage-guide/topics.adoc#deletion[Deletion] for the effect on deleted KafkaTopics. diff --git a/docs/modules/kafka/pages/usage-guide/topics.adoc b/docs/modules/kafka/pages/usage-guide/topics.adoc new file mode 100644 index 000000000..c5501fd80 --- /dev/null +++ b/docs/modules/kafka/pages/usage-guide/topics.adoc @@ -0,0 +1,136 @@ += Topics +:description: Manage Apache Kafka topics declaratively with KafkaTopic resources and the Stackable Operator for Apache Kafka. + +A KafkaTopic declares the desired state of a topic in an Apache Kafka cluster: its name, partitions, replication factor and configs. +The xref:usage-guide/management-access.adoc#_agents[agent] of the cluster creates the topic and keeps it in this state. +This requires xref:usage-guide/management-access.adoc[management access] to be enabled on the KafkaCluster. + +The KafkaTopic is the source of truth for its topic: changes made to the topic outside of the KafkaTopic, for example with `kafka-configs.sh`, are reverted. +The agent checks every topic at least every five minutes. + +NOTE: Other operators, such as Strimzi, also provide a resource of the kind `KafkaTopic`. +If both are installed, use the full resource name with `kubectl`, for example `kubectl get kafkatopics.kafka.stackable.tech`. + +== Configuration + +[source,yaml] +---- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterRef: + name: simple-kafka # <1> + topicName: shop.orders # <2> + partitions: 6 # <3> + replicationFactor: 3 # <4> + configOverrides: # <5> + retention.ms: "604800000" + cleanup.policy: delete + deletionPolicy: Retain # <6> +---- +<1> The KafkaCluster in the same namespace that manages the topic. It cannot be changed. +<2> The name of the topic in Kafka. Defaults to `metadata.name`. It cannot be changed. +<3> The number of partitions. Defaults to the broker setting `num.partitions` when the topic is created. +<4> The number of replicas of each partition. Defaults to the broker setting `default.replication.factor` when the topic is created. +<5> The configs of the topic, see <>. +<6> What happens to the topic when the KafkaTopic is deleted, see <>. + +=== Topic name + +A topic name consists of up to 249 characters out of `a-z`, `A-Z`, `0-9`, `.`, `_` and `-`, and cannot be `.` or `..`. +Kafka allows names that are not valid Kubernetes object names, for example with uppercase letters or underscores. +Such topics need `topicName`. + +Topics starting with `+__+` are internal to Kafka, such as `+__consumer_offsets+`, and are rejected. + +The topic name cannot be changed, and `topicName` can neither be added nor removed later. +To rename a topic, create a new KafkaTopic. + +=== Partitions and replication factor + +The number of partitions can only be increased, because Kafka cannot remove partitions. + +Changing the replication factor of an existing topic is not supported, because Kafka requires an explicit assignment of the replicas to brokers. +Use `kafka-reassign-partitions.sh` or Cruise Control for this. + +An unsupported change is reported in the `Synced` condition, and all other changes are still applied. + +[[configs]] +=== Configs + +`configOverrides` replaces all configs set on the topic. +Configs that are set on the topic but missing in `configOverrides` are reset to the broker defaults. + +The throttles of partition reassignments, `leader.replication.throttled.replicas` and `follower.replication.throttled.replicas`, are left untouched and cannot be set in `configOverrides`. + +== Existing topics + +A KafkaTopic for a topic that already exists in Kafka takes over the topic. + +WARNING: Taking over a topic is destructive. +The configs of the topic are replaced by `configOverrides`, which can delete data, for example with a shorter `retention.ms` or another `cleanup.policy`. +With `deletionPolicy: Delete`, deleting the KafkaTopic also deletes a topic that existed before the KafkaTopic. + +A KafkaTopic refers to its topic by name. +If the topic is deleted and created again outside of the KafkaTopic, the KafkaTopic takes over the new topic as well. + +If several KafkaTopics declare the same topic in the same cluster, the oldest KafkaTopic manages the topic, and the others report a `Conflict`. +When the oldest KafkaTopic is deleted, the next one takes over the topic within five minutes. + +== Status + +The status shows the topic in Kafka: + +[source,yaml] +---- +status: + conditions: ... + topicId: UXyU9i5ARn6W00ON2taeWA # <1> + topicName: shop.orders # <2> + partitions: 6 + replicationFactor: 3 # <3> + config: # <4> + cleanup.policy: delete + retention.ms: "604800000" +---- +<1> The topic ID as shown by `kafka-topics.sh`. +<2> The name of the topic in Kafka. +<3> The replication factor, excluding replicas that a running reassignment removes. It is not set if the partitions have different replication factors. +<4> The configs set on the topic, without broker and default values. + +The `Ready` condition shows whether the topic exists in Kafka. +The `Synced` condition shows whether the topic matches the KafkaTopic. +If it does not, the reason and message of the condition explain why, for example an unsupported change, a request rejected by Kafka, a paused reconciliation or an agent that is not running. + +== Pausing + +The reconciliation of a single KafkaTopic can be paused, independently of the xref:usage-guide/operations/cluster-operations.adoc[cluster operations] of the KafkaCluster: + +[source,yaml] +---- +spec: + topicOperation: + reconciliationPaused: true +---- + +While paused, the agent does not change the topic in Kafka, but still updates the status. + +[[deletion]] +== Deletion + +With the default `deletionPolicy: Retain`, deleting a KafkaTopic leaves the topic in Kafka. + +With `deletionPolicy: Delete`, the agent deletes the topic in Kafka when the KafkaTopic is deleted, unless another KafkaTopic declares the same topic. +The KafkaTopic gets a finalizer once the agent manages the topic, so the deletion waits until the topic is deleted. +If the reconciliation of the KafkaTopic is paused, the deletion waits until it is resumed. +If the brokers cannot be reached, the KafkaTopic reports `ConnectionFailed` and the agent retries the deletion. +Switching the KafkaTopic to `deletionPolicy: Retain` releases the deletion. + +When the KafkaCluster or its namespace is deleted, the operator removes the finalizers of its KafkaTopics, so the deletion does not block. +The topics are not deleted in Kafka in this case. +If the KafkaCluster is created again with the same PersistentVolumeClaims, its topics still exist, but without KafkaTopics. + +If the KafkaCluster exists but has no agent, for example because management access is disabled or the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], the deletion waits for the agent. +In this case, the finalizer `kafka.stackable.tech/kafkatopic` can be removed manually, which leaves the topic in Kafka. diff --git a/docs/modules/kafka/partials/nav.adoc b/docs/modules/kafka/partials/nav.adoc index 7849e974a..07064ed47 100644 --- a/docs/modules/kafka/partials/nav.adoc +++ b/docs/modules/kafka/partials/nav.adoc @@ -6,6 +6,8 @@ ** xref:kafka:usage-guide/listenerclass.adoc[] ** xref:kafka:usage-guide/storage-resources.adoc[] ** xref:kafka:usage-guide/security.adoc[] +** xref:kafka:usage-guide/management-access.adoc[] +** xref:kafka:usage-guide/topics.adoc[] ** xref:kafka:usage-guide/monitoring.adoc[] ** xref:kafka:usage-guide/logging.adoc[] ** xref:kafka:usage-guide/overrides.adoc[] @@ -18,6 +20,7 @@ * xref:kafka:reference/index.adoc[] ** xref:kafka:reference/crds.adoc[] *** {crd-ref}/kafka.stackable.tech/v1alpha1/kafkacluster{crd-ref-release}[KafkaCluster {external-link-icon}^] +*** {crd-ref}/kafka.stackable.tech/v1alpha1/kafkatopic{crd-ref-release}[KafkaTopic {external-link-icon}^] ** xref:kafka:reference/discovery.adoc[] ** xref:kafka:reference/commandline-parameters.adoc[] ** xref:kafka:reference/environment-variables.adoc[] diff --git a/extra/crds.yaml b/extra/crds.yaml index 48821787b..b598f1a02 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2247,3 +2247,179 @@ spec: storage: true subresources: status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kafkatopics.kafka.stackable.tech +spec: + group: kafka.stackable.tech + names: + kind: KafkaTopic + plural: kafkatopics + singular: kafkatopic + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: A Kafka topic, created and updated by the agent of the referenced Kafka cluster. + properties: + spec: + description: |- + A Kafka topic managed by the agent of the referenced Kafka cluster. + + The referenced `KafkaCluster` must have `clusterConfig.managementAccess` enabled or this + resource will not be reconciled. + properties: + clusterRef: + description: The KafkaCluster managing this topic. Immutable. + properties: + name: + description: Name of a KafkaCluster in the namespace of this KafkaTopic. + maxLength: 40 + minLength: 1 + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: clusterRef is immutable + rule: self == oldSelf + configOverrides: + additionalProperties: + type: string + default: {} + description: |- + Topic configs like `retention.ms`. All other topic-level configs are reset to the broker + defaults, except the replication throttles set by partition reassignments. + type: object + deletionPolicy: + default: Retain + description: What happens to a resource in the product when the object managing it is deleted. + enum: + - Retain + - Delete + type: string + partitions: + description: |- + Number of partitions. Can only be increased. Defaults to the broker setting + `num.partitions` when the topic is created. + format: int32 + minimum: 1.0 + nullable: true + type: integer + replicationFactor: + description: |- + Number of replicas of each partition. Changes are not applied to existing topics. + Defaults to the broker setting `default.replication.factor` when the topic is created. + format: int16 + maximum: 32767.0 + minimum: 1.0 + nullable: true + type: integer + topicName: + description: Name of the topic in Kafka. Defaults to `metadata.name`. Immutable. + maxLength: 249 + minLength: 1 + nullable: true + pattern: ^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$ + type: string + x-kubernetes-validations: + - message: topics starting with __ are internal to Kafka + rule: '!self.startsWith(''__'')' + topicOperation: + default: + reconciliationPaused: false + description: Operations on this topic, independent of the `clusterOperation` of the KafkaCluster. + properties: + reconciliationPaused: + default: false + description: Stops the agent from changing the resource in the product. The status is still updated. + type: boolean + type: object + required: + - clusterRef + type: object + x-kubernetes-validations: + - message: topicName is immutable + rule: has(self.topicName) == has(oldSelf.topicName) && (!has(self.topicName) || self.topicName == oldSelf.topicName) + status: + nullable: true + properties: + conditions: + default: [] + description: '`Ready` and `Synced`.' + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + type: string + status: + description: status of the condition, one of True, False, Unknown. + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + config: + additionalProperties: + type: string + default: {} + description: The configs set on the topic in Kafka, without broker and default values. + type: object + partitions: + description: The number of partitions in Kafka. + format: int32 + nullable: true + type: integer + replicationFactor: + description: |- + The replication factor, not counting replicas that a running reassignment removes. + Not set if the partitions have different replication factors. + format: int16 + maximum: 32767.0 + minimum: -32768.0 + nullable: true + type: integer + topicId: + description: The topic id in Kafka, as shown by `kafka-topics.sh`. + nullable: true + type: string + topicName: + description: The name of the topic in Kafka. + maxLength: 249 + minLength: 1 + nullable: true + pattern: ^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$ + type: string + type: object + required: + - spec + title: KafkaTopic + type: object + selectableFields: + - jsonPath: .spec.clusterRef.name + served: true + storage: true + subresources: + status: {} diff --git a/rust/agent-binary/Cargo.toml b/rust/agent-binary/Cargo.toml index c138e675b..eb1fe94cd 100644 --- a/rust/agent-binary/Cargo.toml +++ b/rust/agent-binary/Cargo.toml @@ -9,13 +9,23 @@ repository.workspace = true publish = false [dependencies] +stackable-kafka-admin.workspace = true +stackable-kafka-shared.workspace = true stackable-operator.workspace = true anyhow.workspace = true clap.workspace = true +const_format.workspace = true +futures.workspace = true +serde_json.workspace = true +snafu.workspace = true +strum.workspace = true tokio.workspace = true tracing.workspace = true +[dev-dependencies] +rstest.workspace = true + [build-dependencies] built.workspace = true diff --git a/rust/agent-binary/src/config.rs b/rust/agent-binary/src/config.rs new file mode 100644 index 000000000..9c2dcc47d --- /dev/null +++ b/rust/agent-binary/src/config.rs @@ -0,0 +1,49 @@ +//! The agent's view of the files and settings the operator provides. + +use const_format::concatcp; +use stackable_kafka_admin::connection::Security; +use stackable_kafka_shared::agent::{ClientSecurity, KAFKA_TLS_DIR}; +use stackable_operator::crd::management_access::tls::MOUNT_PATH; + +const CA_FILE: &str = concatcp!(KAFKA_TLS_DIR, "/ca.crt"); +const CERTIFICATE_FILE: &str = concatcp!(MOUNT_PATH, "/tls.crt"); +const KEY_FILE: &str = concatcp!(MOUNT_PATH, "/tls.key"); + +pub fn security(client_security: ClientSecurity) -> Security { + match client_security { + ClientSecurity::Plaintext => Security::Plaintext, + ClientSecurity::Tls => Security::Tls { ca: CA_FILE.into() }, + ClientSecurity::MutualTls => Security::MutualTls { + ca: CA_FILE.into(), + certificate: CERTIFICATE_FILE.into(), + key: KEY_FILE.into(), + }, + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::plaintext(ClientSecurity::Plaintext, Security::Plaintext)] + #[case::tls( + ClientSecurity::Tls, + Security::Tls { + ca: "/stackable/secrets/tls-kafka/ca.crt".into(), + } + )] + #[case::mutual_tls( + ClientSecurity::MutualTls, + Security::MutualTls { + ca: "/stackable/secrets/tls-kafka/ca.crt".into(), + certificate: "/stackable/secrets/tls-management-access/tls.crt".into(), + key: "/stackable/secrets/tls-management-access/tls.key".into(), + } + )] + fn map_client_security(#[case] client_security: ClientSecurity, #[case] expected: Security) { + assert_eq!(expected, security(client_security)); + } +} diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 0934f803f..48efbb2d3 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -1,12 +1,45 @@ //! The Stackable Kafka agent: a per-cluster controller that reconciles in-cluster resources like `KafkaTopic`s. +#![allow(clippy::large_enum_variant, clippy::result_large_err)] + +use std::sync::Arc; + use clap::Parser; +use futures::StreamExt; +use stackable_kafka_admin::{bootstrap::parse_bootstrap_servers, client::KafkaAdminClient}; +use stackable_kafka_shared::{ + agent::{ + ClientSecurity, ENV_BOOTSTRAP_SERVERS, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, + ENV_CLUSTER_NAMESPACE, + }, + crd::topic::v1alpha1::KafkaTopic, +}; use stackable_operator::{ cli::{Command, CommonOptions}, + client, + kube::{ + core::DeserializeGuard, + runtime::{ + Controller, + events::{Recorder, Reporter}, + watcher, + }, + }, + logging::controller::report_controller_reconciled, telemetry::Tracing, utils::signal::SignalWatcher, + v2::types::{kubernetes::NamespaceName, operator::ClusterName}, }; +use crate::topic::controller::{self, TOPIC_FULL_CONTROLLER_NAME}; + +mod config; +mod topic; + +pub const AGENT_NAME: &str = "kafka-agent.stackable.tech"; + +const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; + mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } @@ -15,14 +48,46 @@ mod built_info { #[clap(about, author)] struct Opts { #[clap(subcommand)] - cmd: Command, + cmd: Command, +} + +#[derive(clap::Parser)] +struct KafkaAgentRun { + /// Name of the KafkaCluster this agent belongs to. + #[arg(long, env = AsRef::::as_ref(&*ENV_CLUSTER_NAME))] + cluster_name: ClusterName, + + /// Namespace of the KafkaCluster. + #[arg(long, env = AsRef::::as_ref(&*ENV_CLUSTER_NAMESPACE))] + cluster_namespace: NamespaceName, + + /// How to connect to the brokers: plaintext, tls or mutual-tls. + #[arg(long, env = AsRef::::as_ref(&*ENV_CLIENT_SECURITY))] + client_security: ClientSecurity, + + /// The brokers to connect to, as `host:port,host:port`. + #[arg(long, env = AsRef::::as_ref(&*ENV_BOOTSTRAP_SERVERS))] + bootstrap_servers: String, + + #[clap(flatten)] + common: CommonOptions, } #[tokio::main] async fn main() -> anyhow::Result<()> { match Opts::parse().cmd { Command::Crd => anyhow::bail!("this agent has no CRDs, they are owned by the operator"), - Command::Run(CommonOptions { telemetry, .. }) => { + Command::Run(KafkaAgentRun { + cluster_name, + cluster_namespace, + client_security, + bootstrap_servers, + common: + CommonOptions { + telemetry, + cluster_info, + }, + }) => { let _tracing_guard = Tracing::pre_configured(built_info::PKG_NAME, telemetry).init()?; tracing::info!( @@ -35,8 +100,65 @@ async fn main() -> anyhow::Result<()> { description = built_info::PKG_DESCRIPTION ); + let security = config::security(client_security); + let bootstrap_servers = parse_bootstrap_servers(&bootstrap_servers)?; + tracing::info!( + %cluster_name, + %cluster_namespace, + ?security, + ?bootstrap_servers, + "Configured the agent" + ); + let admin_client = KafkaAdminClient::new(bootstrap_servers, security, KAFKA_CLIENT_ID); + + // Watches for the SIGTERM signal and gracefully shuts down the controller. let sigterm_watcher = SignalWatcher::sigterm()?; - sigterm_watcher.handle().await; + + let client = + client::initialize_operator(Some(AGENT_NAME.to_owned()), &cluster_info).await?; + + let event_recorder = Arc::new(Recorder::new( + client.as_kube_client(), + Reporter { + controller: TOPIC_FULL_CONTROLLER_NAME.to_string(), + instance: None, + }, + )); + + let topic_controller = Controller::new( + client.get_api::>(cluster_namespace.as_ref()), + // Works because the CRD declares the field as a selectableField. + watcher::Config::default().fields(&format!("spec.clusterRef.name={cluster_name}")), + ); + let kafka_topic_store = topic_controller.store(); + topic_controller + .graceful_shutdown_on(sigterm_watcher.handle()) + .run( + controller::reconcile_topic, + controller::error_policy, + Arc::new(controller::Ctx::new( + client, + admin_client, + kafka_topic_store, + )), + ) + .for_each_concurrent( + 16, // concurrency limit + move |result| { + // The event_recorder needs to be shared across all invocations, so that + // events are correctly aggregated + let event_recorder = event_recorder.clone(); + async move { + report_controller_reconciled( + &event_recorder, + TOPIC_FULL_CONTROLLER_NAME, + &result, + ) + .await; + } + }, + ) + .await; } } diff --git a/rust/agent-binary/src/topic/condition.rs b/rust/agent-binary/src/topic/condition.rs new file mode 100644 index 000000000..ccdba66be --- /dev/null +++ b/rust/agent-binary/src/topic/condition.rs @@ -0,0 +1,384 @@ +//! The `Ready` and `Synced` conditions of a `KafkaTopic` for the outcome of a sync. + +use stackable_kafka_admin::client; +use stackable_operator::agent::condition::{ + ConditionStatus, ManagedCondition, ManagedConditionType, +}; + +use super::plan::{TopicState, UnsupportedChange}; + +#[derive(Debug, Eq, PartialEq)] +pub enum TopicSyncOutcome { + /// The topic matches the spec + Synced { + reassigning_partition_count: usize, + }, + /// The topic matches the spec, apart from the changes the agent does not apply + UnsupportedChange { + changes: Vec, + }, + ReconciliationPaused, + InvalidSpec { + message: String, + }, + /// Another `KafkaTopic` manages the same topic + Conflict { + message: String, + }, + KafkaRejected { + message: String, + }, + ConnectionFailed { + message: String, + }, +} + +impl TopicSyncOutcome { + pub fn from_client_error(error: &client::Error) -> Self { + let message = error_chain(error); + if error.is_rejected_by_kafka() { + TopicSyncOutcome::KafkaRejected { message } + } else { + TopicSyncOutcome::ConnectionFailed { message } + } + } +} + +/// `topic_state` is the topic in Kafka, if it exists and is managed by this `KafkaTopic`. +pub fn conditions_from( + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> Vec { + vec![ + ready_from(sync_outcome, topic_state), + synced_from(sync_outcome, topic_state), + ] +} + +/// Whether the topic exists in Kafka, `Unknown` if the agent could not find out. +fn ready_from( + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> ManagedCondition { + use ConditionStatus::{False, True, Unknown}; + + // Also after a failed change, because the topic was observed before. + if let Some(topic_state) = topic_state { + return ready( + True, + "Available", + &format!( + "Topic {} exists with {} partitions", + topic_state.name, topic_state.partition_count + ), + ); + } + match sync_outcome { + TopicSyncOutcome::Synced { .. } + | TopicSyncOutcome::UnsupportedChange { .. } + | TopicSyncOutcome::ReconciliationPaused => { + ready(False, "Unavailable", "the topic does not exist in Kafka") + } + TopicSyncOutcome::Conflict { .. } => { + ready(False, "Conflict", "another KafkaTopic manages the topic") + } + TopicSyncOutcome::InvalidSpec { message } => ready(Unknown, "InvalidSpec", message), + TopicSyncOutcome::KafkaRejected { message } => ready(Unknown, "KafkaRejected", message), + TopicSyncOutcome::ConnectionFailed { message } => { + ready(Unknown, "ConnectionFailed", message) + } + } +} + +/// Whether the last sync applied the spec to the topic. +fn synced_from( + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> ManagedCondition { + use ConditionStatus::{False, True}; + + match sync_outcome { + TopicSyncOutcome::Synced { + reassigning_partition_count, + } => match (reassigning_partition_count, topic_state) { + (0, _) | (_, None) => synced(True, "Synced", "the topic matches the spec"), + (reassigning, Some(topic_state)) => synced( + True, + "ReassignmentInProgress", + &format!( + "{reassigning} of {} partitions are being reassigned", + topic_state.partition_count + ), + ), + }, + TopicSyncOutcome::UnsupportedChange { changes } => { + let messages = changes.iter().map(message_from).collect::>(); + synced(False, "UnsupportedChange", &messages.join("; ")) + } + TopicSyncOutcome::ReconciliationPaused => synced( + False, + "ReconciliationPaused", + "the reconciliation is paused, only the status is updated", + ), + TopicSyncOutcome::Conflict { message } => synced(False, "Conflict", message), + TopicSyncOutcome::InvalidSpec { message } => synced(False, "InvalidSpec", message), + TopicSyncOutcome::KafkaRejected { message } => synced(False, "KafkaRejected", message), + TopicSyncOutcome::ConnectionFailed { message } => { + synced(False, "ConnectionFailed", message) + } + } +} + +fn message_from(unsupported_change: &UnsupportedChange) -> String { + match unsupported_change { + UnsupportedChange::DecreasePartitions { desired, actual } => { + format!("partitions cannot be decreased (spec: {desired}, actual: {actual})") + } + UnsupportedChange::ChangeReplicationFactor { + desired, + actual_min, + actual_max, + } => { + let actual = if actual_min == actual_max { + actual_min.to_string() + } else { + format!("between {actual_min} and {actual_max}") + }; + format!( + "the replication factor cannot be changed (spec: {desired}, actual: {actual}); \ + change it with kafka-reassign-partitions" + ) + } + } +} + +/// Renders the error and its sources, skipping a source whose message the previous error already +/// contains. +fn error_chain(error: &dyn std::error::Error) -> String { + snafu::CleanedErrorText::new(error) + .map(|(_, text, _)| text) + .filter(|text| !text.is_empty()) + .collect::>() + .join(": ") +} + +fn ready(status: ConditionStatus, reason: &str, message: &str) -> ManagedCondition { + condition(ManagedConditionType::Ready, status, reason, message) +} + +fn synced(status: ConditionStatus, reason: &str, message: &str) -> ManagedCondition { + condition(ManagedConditionType::Synced, status, reason, message) +} + +fn condition( + type_: ManagedConditionType, + status: ConditionStatus, + reason: &str, + message: &str, +) -> ManagedCondition { + ManagedCondition { + type_, + status, + reason: reason.to_owned(), + message: message.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, str::FromStr}; + + use snafu::Snafu; + use stackable_kafka_shared::crd::topic::TopicName; + + use super::*; + use crate::topic::condition::ConditionStatus::{False, True, Unknown}; + + fn summary(conditions: &[ManagedCondition]) -> Vec<(String, ConditionStatus, &str)> { + conditions + .iter() + .map(|condition| { + ( + condition.type_.to_string(), + condition.status, + condition.reason.as_str(), + ) + }) + .collect() + } + + fn topic_state(partition_count: i32) -> TopicState { + TopicState { + name: TopicName::from_str("orders").expect("the topic name should be valid"), + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partition_count, + replication_factor: Some(1), + topic_configs: BTreeMap::new(), + } + } + + #[test] + fn mark_a_synced_topic() { + let conditions = conditions_from( + &TopicSyncOutcome::Synced { + reassigning_partition_count: 0, + }, + Some(&topic_state(6)), + ); + + assert_eq!( + vec![ + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), True, "Synced") + ], + summary(&conditions) + ); + assert_eq!( + "Topic orders exists with 6 partitions", + conditions[0].message + ); + } + + #[test] + fn keep_a_reassigned_topic_synced() { + let conditions = conditions_from( + &TopicSyncOutcome::Synced { + reassigning_partition_count: 3, + }, + Some(&topic_state(6)), + ); + + assert_eq!( + vec![ + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), True, "ReassignmentInProgress") + ], + summary(&conditions) + ); + assert_eq!( + "3 of 6 partitions are being reassigned", + conditions[1].message + ); + } + + #[test] + fn mark_an_unsupported_change() { + let conditions = conditions_from( + &TopicSyncOutcome::UnsupportedChange { + changes: vec![ + UnsupportedChange::DecreasePartitions { + desired: 4, + actual: 6, + }, + UnsupportedChange::ChangeReplicationFactor { + desired: 2, + actual_min: 2, + actual_max: 3, + }, + ], + }, + Some(&topic_state(6)), + ); + + assert_eq!( + vec![ + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), False, "UnsupportedChange") + ], + summary(&conditions) + ); + assert_eq!( + "partitions cannot be decreased (spec: 4, actual: 6); the replication factor cannot \ + be changed (spec: 2, actual: between 2 and 3); change it with \ + kafka-reassign-partitions", + conditions[1].message + ); + } + + #[test] + fn report_a_missing_topic_while_paused() { + let conditions = conditions_from(&TopicSyncOutcome::ReconciliationPaused, None); + + assert_eq!( + vec![ + ("Ready".to_owned(), False, "Unavailable"), + ("Synced".to_owned(), False, "ReconciliationPaused") + ], + summary(&conditions) + ); + } + + #[test] + fn mark_a_conflict_as_not_ready() { + let conditions = conditions_from( + &TopicSyncOutcome::Conflict { + message: "managed by older".to_owned(), + }, + None, + ); + + assert_eq!( + vec![ + ("Ready".to_owned(), False, "Conflict"), + ("Synced".to_owned(), False, "Conflict") + ], + summary(&conditions) + ); + } + + #[rstest::rstest] + #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: "invalid".to_owned() }, "InvalidSpec")] + #[case::kafka_rejected(TopicSyncOutcome::KafkaRejected { message: "denied".to_owned() }, "KafkaRejected")] + #[case::connection_failed(TopicSyncOutcome::ConnectionFailed { message: "timed out".to_owned() }, "ConnectionFailed")] + fn mark_ready_as_unknown_if_the_topic_cannot_be_observed( + #[case] sync_outcome: TopicSyncOutcome, + #[case] reason: &str, + ) { + assert_eq!( + vec![ + ("Ready".to_owned(), Unknown, reason), + ("Synced".to_owned(), False, reason) + ], + summary(&conditions_from(&sync_outcome, None)) + ); + } + + #[test] + fn keep_an_observed_topic_ready_if_a_change_fails() { + let conditions = conditions_from( + &TopicSyncOutcome::KafkaRejected { + message: "denied".to_owned(), + }, + Some(&topic_state(6)), + ); + + assert_eq!( + vec![ + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), False, "KafkaRejected") + ], + summary(&conditions) + ); + } + + #[derive(Debug, Snafu)] + enum Outer { + #[snafu(display("failed to connect to the bootstrap server kafka-0:9093"))] + Connect { source: Inner }, + } + + #[derive(Debug, Snafu)] + #[snafu(display("invalid peer certificate: UnknownIssuer"))] + struct Inner; + + #[test] + fn render_the_error_chain() { + let error = Outer::Connect { source: Inner }; + + assert_eq!( + "failed to connect to the bootstrap server kafka-0:9093: invalid peer certificate: \ + UnknownIssuer", + error_chain(&error) + ); + } +} diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs new file mode 100644 index 000000000..e9b0f7f6a --- /dev/null +++ b/rust/agent-binary/src/topic/controller.rs @@ -0,0 +1,183 @@ +//! The `KafkaTopic` controller: reconcile and error policy. + +use std::{sync::Arc, time::Duration}; + +use const_format::concatcp; +use snafu::{ResultExt, Snafu}; +use stackable_kafka_admin::client::{self, KafkaAdminClient}; +use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; +use stackable_operator::{ + kube::{ + core::{DeserializeGuard, error_boundary}, + runtime::{controller::Action, reflector::Store}, + }, + logging::controller::ReconcilerError, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; +use tokio::sync::Mutex; +use tracing::info; + +use super::{ + condition::TopicSyncOutcome, + finalizer::{add_finalizer, reconcile_finalizer}, + plan::TopicState, + sync::sync, + update_status::update_status, +}; +use crate::AGENT_NAME; + +pub const TOPIC_CONTROLLER_NAME: &str = "kafkatopic"; +pub const TOPIC_FULL_CONTROLLER_NAME: &str = concatcp!(TOPIC_CONTROLLER_NAME, ".", AGENT_NAME); + +const RESYNC_INTERVAL: Duration = Duration::from_secs(5 * 60); +const ERROR_RETRY_INTERVAL: Duration = Duration::from_secs(30); + +#[derive(Debug, Snafu, EnumDiscriminants)] +#[snafu(visibility(pub(crate)))] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("the KafkaTopic is invalid"))] + InvalidKafkaTopic { + source: error_boundary::InvalidObject, + }, + + #[snafu(display("failed to talk to Kafka"))] + Kafka { source: client::Error }, + + #[snafu(display("failed to update the status of the KafkaTopic"))] + UpdateStatus { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to update the finalizers of the KafkaTopic"))] + UpdateFinalizers { + source: stackable_operator::client::Error, + }, +} + +pub(crate) type Result = std::result::Result; + +impl ReconcilerError for Error { + fn category(&self) -> &'static str { + ErrorDiscriminants::from(self).into() + } +} + +pub struct Ctx { + pub(crate) client: stackable_operator::client::Client, + pub(crate) kafka_topics: Store>, + /// Reads the TLS files on every new connection and reconnects after a failed request, so + /// rotated certificates are picked up without restarting the agent. The Kafka requests of + /// all reconciles are serialized on its single connection. + pub(crate) admin_client: Mutex, +} + +impl Ctx { + /// `kafka_topics` is the store of the controller, used to detect conflicts between + /// `KafkaTopic`s. + pub fn new( + client: stackable_operator::client::Client, + admin_client: KafkaAdminClient, + kafka_topics: Store>, + ) -> Self { + Self { + client, + kafka_topics, + admin_client: Mutex::new(admin_client), + } + } +} + +pub async fn reconcile_topic( + kafka_topic: Arc>, + ctx: Arc, +) -> Result { + info!("Starting reconcile"); + + // dereference + let kafka_topic = kafka_topic + .0 + .as_ref() + .map_err(error_boundary::InvalidObject::clone) + .context(InvalidKafkaTopicSnafu)?; + + // finalizer (Kubernetes) + // Left untouched while paused, like operators that only update the status then. A deletion + // waits until the reconciliation is resumed. + let paused = kafka_topic.spec.topic_operation.reconciliation_paused; + if !paused { + match reconcile_finalizer(kafka_topic, &ctx).await { + Ok(Some(action)) => return Ok(action), + Ok(None) => {} + Err(error) => return fail_with_status(kafka_topic, &ctx, error, None).await, + } + } + + // sync (Kafka) + let mut topic_state = None; + let sync_result = sync(kafka_topic, &ctx, &mut topic_state).await; + + // finalizer (Kubernetes) + // Added before the status, because the status patch changes the resourceVersion. + if !paused { + add_finalizer(kafka_topic, &ctx, topic_state.as_ref()).await?; + } + + // update status (Kubernetes) + match sync_result { + Ok(sync_outcome) => { + update_status(kafka_topic, &ctx, &sync_outcome, topic_state.as_ref()).await?; + Ok(requeue(&sync_outcome)) + } + Err(error) => fail_with_status(kafka_topic, &ctx, error, topic_state.as_ref()).await, + } +} + +/// Reports a Kafka error in the status of the `KafkaTopic` before returning it. Other errors are +/// returned as they are. +async fn fail_with_status( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + error: Error, + topic_state: Option<&TopicState>, +) -> Result { + if let Error::Kafka { source } = &error { + let sync_outcome = TopicSyncOutcome::from_client_error(source); + update_status(kafka_topic, ctx, &sync_outcome, topic_state).await?; + } + Err(error) +} + +fn requeue(sync_outcome: &TopicSyncOutcome) -> Action { + match sync_outcome { + TopicSyncOutcome::InvalidSpec { .. } => Action::await_change(), + _ => Action::requeue(RESYNC_INTERVAL), + } +} + +pub fn error_policy( + _kafka_topic: Arc>, + error: &Error, + _ctx: Arc, +) -> Action { + match error { + Error::InvalidKafkaTopic { .. } => Action::await_change(), + _ => Action::requeue(ERROR_RETRY_INTERVAL), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[rstest::rstest] + #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: String::new() }, Action::await_change())] + #[case::synced(TopicSyncOutcome::Synced { reassigning_partition_count: 0 }, Action::requeue(RESYNC_INTERVAL))] + #[case::paused( + TopicSyncOutcome::ReconciliationPaused, + Action::requeue(RESYNC_INTERVAL) + )] + fn requeue_by_outcome(#[case] sync_outcome: TopicSyncOutcome, #[case] expected: Action) { + assert_eq!(expected, requeue(&sync_outcome)); + } +} diff --git a/rust/agent-binary/src/topic/finalizer.rs b/rust/agent-binary/src/topic/finalizer.rs new file mode 100644 index 000000000..1eb9fc39d --- /dev/null +++ b/rust/agent-binary/src/topic/finalizer.rs @@ -0,0 +1,126 @@ +//! The finalizer of a `KafkaTopic` and the cleanup in Kafka when it is deleted. + +use serde_json::json; +use snafu::ResultExt; +use stackable_kafka_shared::crd::topic::{KAFKA_TOPIC_FINALIZER, v1alpha1::KafkaTopic}; +use stackable_operator::{ + agent::deletion::v1alpha1::DeletionPolicy, + kube::{ResourceExt, runtime::controller::Action}, +}; +use tracing::info; + +use super::{ + controller::{Ctx, KafkaSnafu, Result, UpdateFinalizersSnafu}, + plan::TopicState, + validate::has_other_kafka_topic, +}; + +/// Handles a deletion and removes a finalizer that is no longer needed. Returns `Some` if the +/// reconcile ends here. +pub(crate) async fn reconcile_finalizer( + kafka_topic: &KafkaTopic, + ctx: &Ctx, +) -> Result> { + let has_finalizer = has_finalizer(kafka_topic); + + if kafka_topic.metadata.deletion_timestamp.is_some() { + if has_finalizer { + cleanup(kafka_topic, ctx).await?; + set_finalizer(kafka_topic, ctx, false).await?; + } + return Ok(Some(Action::await_change())); + } + + if has_finalizer && kafka_topic.spec.deletion_policy != DeletionPolicy::Delete { + // The patch triggers another reconcile with the updated object. + set_finalizer(kafka_topic, ctx, false).await?; + return Ok(Some(Action::await_change())); + } + Ok(None) +} + +/// Adds the finalizer only once the topic exists and is managed by this `KafkaTopic`, so deleting +/// a `KafkaTopic` that never managed the topic, e.g. because of an invalid spec or a conflict, +/// does not delete it. +pub(crate) async fn add_finalizer( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + topic_state: Option<&TopicState>, +) -> Result<()> { + if kafka_topic.spec.deletion_policy == DeletionPolicy::Delete + && topic_state.is_some() + && !has_finalizer(kafka_topic) + { + set_finalizer(kafka_topic, ctx, true).await?; + } + Ok(()) +} + +fn has_finalizer(kafka_topic: &KafkaTopic) -> bool { + kafka_topic + .finalizers() + .iter() + .any(|finalizer| finalizer == KAFKA_TOPIC_FINALIZER) +} + +/// Deletes the topic in Kafka if the `deletionPolicy` says so and no other `KafkaTopic` declares +/// it. +/// +/// The policy is read from the `KafkaTopic` being deleted, so switching it to `Retain` releases a +/// deletion that is stuck because Kafka cannot be reached. +async fn cleanup(kafka_topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + if kafka_topic.spec.deletion_policy != DeletionPolicy::Delete { + return Ok(()); + } + // A topic with an invalid name was never created. + let Ok(topic_name) = kafka_topic.topic_name() else { + return Ok(()); + }; + if has_other_kafka_topic(kafka_topic, &topic_name, &ctx.kafka_topics) { + return Ok(()); + } + + ctx.admin_client + .lock() + .await + .delete_topic(topic_name.as_ref()) + .await + .context(KafkaSnafu)?; + + info!( + topic = %topic_name, + kafka_topic = %kafka_topic.name_any(), + "Deleted the topic" + ); + + Ok(()) +} + +/// Fails if the `KafkaTopic` was changed since it was read, so concurrent changes to the +/// finalizers are not lost. +async fn set_finalizer( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + should_have_finalizer: bool, +) -> Result<()> { + let mut finalizers = kafka_topic + .finalizers() + .iter() + .filter(|finalizer| *finalizer != KAFKA_TOPIC_FINALIZER) + .cloned() + .collect::>(); + if should_have_finalizer { + finalizers.push(KAFKA_TOPIC_FINALIZER.to_owned()); + } + let patch = json!({ + "metadata": { + "resourceVersion": kafka_topic.metadata.resource_version, + "finalizers": finalizers, + } + }); + ctx.client + .merge_patch(kafka_topic, patch) + .await + .context(UpdateFinalizersSnafu)?; + Ok(()) +} diff --git a/rust/agent-binary/src/topic/mod.rs b/rust/agent-binary/src/topic/mod.rs new file mode 100644 index 000000000..062a1e7aa --- /dev/null +++ b/rust/agent-binary/src/topic/mod.rs @@ -0,0 +1,11 @@ +//! Reconciles `KafkaTopic`s into topics in Kafka. + +mod condition; +pub mod controller; +mod finalizer; +mod plan; +mod sync; +#[cfg(test)] +mod test_support; +mod update_status; +mod validate; diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs new file mode 100644 index 000000000..a204f137e --- /dev/null +++ b/rust/agent-binary/src/topic/plan.rs @@ -0,0 +1,466 @@ +//! What the agent has to do in Kafka to bring a topic to the state of its `KafkaTopic` spec. + +use std::collections::{BTreeMap, BTreeSet}; + +use stackable_kafka_admin::client::{CreatedTopic, PartitionReassignment, Topic}; +use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopicSpec}; + +/// Set by `kafka-reassign-partitions` and Cruise Control for the duration of a reassignment. +/// Resetting them would lift the throttle of a running reassignment. +pub const THROTTLE_CONFIG_KEYS: [&str; 2] = [ + "leader.replication.throttled.replicas", + "follower.replication.throttled.replicas", +]; + +/// The state of an existing topic in Kafka. +pub struct ObservedTopic { + pub topic: Topic, + pub topic_configs: BTreeMap, + pub reassignments: Vec, +} + +/// The state of a topic in Kafka as shown in the status. +#[derive(Debug, Eq, PartialEq)] +pub struct TopicState { + pub name: TopicName, + pub id: String, + pub partition_count: i32, + pub replication_factor: Option, + pub topic_configs: BTreeMap, +} + +impl TopicState { + pub fn observed(name: TopicName, observed: &ObservedTopic) -> Self { + Self { + name, + id: observed.topic.id.clone(), + partition_count: observed.topic.partitions.len() as i32, + replication_factor: replication_factor(observed), + topic_configs: observed.topic_configs.clone(), + } + } + + pub fn created(name: TopicName, created: CreatedTopic) -> Self { + Self { + name, + id: created.id, + partition_count: created.partitions, + replication_factor: Some(created.replication_factor), + topic_configs: created.configs, + } + } + + /// The topic-level configs as they are after the changes are applied. + pub fn apply_config_changes( + &mut self, + configs_to_set: &BTreeMap, + configs_to_reset: &BTreeSet, + ) { + self.topic_configs + .retain(|key, _| !configs_to_reset.contains(key)); + self.topic_configs.extend( + configs_to_set + .iter() + .map(|(key, value)| (key.clone(), value.clone())), + ); + } +} + +#[derive(Debug, Eq, PartialEq)] +pub enum TopicPlan { + Create { + partition_count: Option, + replication_factor: Option, + config_overrides: BTreeMap, + }, + Update { + topic_update: TopicUpdate, + /// Applied are only the changes in `topic_update` + unsupported_changes: Vec, + }, +} + +/// Changes to an existing topic. No changes means the topic matches the spec, apart from +/// [`UnsupportedChange`]s. +#[derive(Debug, Default, Eq, PartialEq)] +pub struct TopicUpdate { + /// Only increased, see [`UnsupportedChange::DecreasePartitions`] + pub new_partition_count: Option, + pub configs_to_set: BTreeMap, + /// Topic-level configs to remove, so that the broker configs apply + pub configs_to_reset: BTreeSet, +} + +/// A difference to the spec the agent does not apply. The other changes are still applied. +#[derive(Debug, Eq, PartialEq)] +pub enum UnsupportedChange { + /// Kafka cannot remove partitions. + DecreasePartitions { desired: i32, actual: i32 }, + /// The partitions have between `actual_min` and `actual_max` replicas, not counting replicas + /// that a running reassignment removes. + /// + /// Kafka can only change the replicas of existing partitions with + /// `AlterPartitionReassignments` and explicit replica lists. Choosing the brokers is left to + /// `kafka-reassign-partitions` or Cruise Control. + ChangeReplicationFactor { + desired: i16, + actual_min: usize, + actual_max: usize, + }, +} + +pub fn plan(spec: &KafkaTopicSpec, observed: Option<&ObservedTopic>) -> TopicPlan { + let Some(observed) = observed else { + return TopicPlan::Create { + partition_count: spec.partitions, + replication_factor: spec.replication_factor, + config_overrides: spec.config_overrides.clone(), + }; + }; + + let actual_partition_count = observed.topic.partitions.len() as i32; + let (configs_to_set, configs_to_reset) = + config_changes(&spec.config_overrides, &observed.topic_configs); + + TopicPlan::Update { + topic_update: TopicUpdate { + new_partition_count: spec + .partitions + .filter(|desired| *desired > actual_partition_count), + configs_to_set, + configs_to_reset, + }, + unsupported_changes: unsupported_changes(spec, observed), + } +} + +fn unsupported_changes(spec: &KafkaTopicSpec, observed: &ObservedTopic) -> Vec { + let mut changes = Vec::new(); + + let actual_partition_count = observed.topic.partitions.len() as i32; + if let Some(desired) = spec.partitions + && desired < actual_partition_count + { + changes.push(UnsupportedChange::DecreasePartitions { + desired, + actual: actual_partition_count, + }); + } + + let replication_factors = target_replication_factors(observed); + if let Some(desired) = spec.replication_factor + && let (Some(&actual_min), Some(&actual_max)) = + (replication_factors.first(), replication_factors.last()) + && (actual_min != actual_max || actual_min != desired as usize) + { + changes.push(UnsupportedChange::ChangeReplicationFactor { + desired, + actual_min, + actual_max, + }); + } + + changes +} + +/// The configs to set and the configs to reset, so that the topic-level configs are exactly the +/// desired ones. The throttles of partition reassignments are not reset. +fn config_changes( + desired_topic_configs: &BTreeMap, + current_topic_configs: &BTreeMap, +) -> (BTreeMap, BTreeSet) { + // A desired value that equals the broker config is still set on the topic, so a later change of + // the broker config does not apply to the topic. Unchanged values are skipped, so a resync + // without drift does not write to Kafka. + let configs_to_set = desired_topic_configs + .iter() + .filter(|(key, value)| current_topic_configs.get(*key) != Some(value)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + let configs_to_reset = current_topic_configs + .keys() + .filter(|key| { + !desired_topic_configs.contains_key(*key) + && !THROTTLE_CONFIG_KEYS.contains(&key.as_str()) + }) + .cloned() + .collect(); + (configs_to_set, configs_to_reset) +} + +/// The replication factor of the partitions, excluding replicas a reassignment removes. `None` if +/// the partitions have different replication factors. +fn replication_factor(observed: &ObservedTopic) -> Option { + match target_replication_factors(observed).as_slice() { + [replication_factor] => i16::try_from(*replication_factor).ok(), + _ => None, + } +} + +/// The distinct replication factors of the partitions in ascending order, excluding replicas a +/// reassignment removes. +/// +/// Kafka stores no replication factor per topic. During a reassignment, the replicas of a +/// partition include the ones being added and removed. `kafka-topics.sh --describe` shows the +/// replicas minus the ones being added of the first partition instead. +fn target_replication_factors(observed: &ObservedTopic) -> Vec { + observed + .topic + .partitions + .iter() + .map(|partition| { + let removing = observed + .reassignments + .iter() + .find(|reassignment| reassignment.index == partition.index) + .map_or(0, |reassignment| reassignment.removing_replicas); + partition.replicas.saturating_sub(removing) + }) + .collect::>() + .into_iter() + .collect() +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use stackable_kafka_admin::client::Partition; + + use super::*; + use crate::topic::test_support::kafka_topic; + + fn spec( + partitions: Option, + replication_factor: Option, + config: &[(&str, &str)], + ) -> KafkaTopicSpec { + let mut spec = kafka_topic("orders", Some("orders")).spec; + spec.partitions = partitions; + spec.replication_factor = replication_factor; + spec.config_overrides = configs(config); + spec + } + + fn configs(entries: &[(&str, &str)]) -> BTreeMap { + entries + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect() + } + + fn keys(keys: &[&str]) -> BTreeSet { + keys.iter().map(|key| key.to_string()).collect() + } + + fn update(topic_update: TopicUpdate, unsupported_changes: Vec) -> TopicPlan { + TopicPlan::Update { + topic_update, + unsupported_changes, + } + } + + /// A topic whose partitions have the given number of replicas, with the given topic-level + /// configs + fn observed(replicas: &[usize], topic_configs: &[(&str, &str)]) -> ObservedTopic { + ObservedTopic { + topic: Topic { + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partitions: replicas + .iter() + .zip(0..) + .map(|(replicas, index)| Partition { + index, + replicas: *replicas, + }) + .collect(), + }, + topic_configs: configs(topic_configs), + reassignments: vec![], + } + } + + #[test] + fn create_missing_topic() { + let plan = plan(&spec(Some(3), None, &[("retention.ms", "1000")]), None); + + assert_eq!( + TopicPlan::Create { + partition_count: Some(3), + replication_factor: None, + config_overrides: configs(&[("retention.ms", "1000")]), + }, + plan + ); + } + + #[test] + fn do_nothing_for_a_matching_topic() { + let observed = observed(&[2, 2], &[("retention.ms", "1000")]); + let spec = spec(Some(2), Some(2), &[("retention.ms", "1000")]); + + assert_eq!( + update(TopicUpdate::default(), vec![]), + plan(&spec, Some(&observed)) + ); + } + + #[test] + fn leave_partitions_and_replication_factor_alone_if_not_set() { + let observed = observed(&[3, 3, 3], &[]); + let spec = spec(None, None, &[]); + + assert_eq!( + update(TopicUpdate::default(), vec![]), + plan(&spec, Some(&observed)) + ); + } + + #[test] + fn grow_partitions() { + let observed = observed(&[1, 1], &[]); + + let plan = plan(&spec(Some(4), Some(1), &[]), Some(&observed)); + + assert_eq!( + update( + TopicUpdate { + new_partition_count: Some(4), + ..TopicUpdate::default() + }, + vec![] + ), + plan + ); + } + + #[test] + fn report_fewer_partitions_as_unsupported() { + let observed = observed(&[1, 1, 1], &[]); + + let plan = plan(&spec(Some(2), None, &[]), Some(&observed)); + + assert_eq!( + update( + TopicUpdate::default(), + vec![UnsupportedChange::DecreasePartitions { + desired: 2, + actual: 3 + }] + ), + plan + ); + } + + #[test] + fn replace_the_topic_level_configs() { + let observed = observed( + &[1], + &[ + ("retention.ms", "1000"), + ("cleanup.policy", "compact"), + ("leader.replication.throttled.replicas", "0:1"), + ], + ); + + let plan = plan( + &spec( + None, + None, + &[("retention.ms", "2000"), ("max.message.bytes", "10")], + ), + Some(&observed), + ); + + assert_eq!( + update( + TopicUpdate { + configs_to_set: configs(&[ + ("max.message.bytes", "10"), + ("retention.ms", "2000") + ]), + // The throttles belong to a reassignment + configs_to_reset: keys(&["cleanup.policy"]), + ..TopicUpdate::default() + }, + vec![] + ), + plan + ); + } + + #[test] + fn apply_config_changes_to_the_topic_state() { + let mut topic_state = TopicState::observed( + TopicName::from_str("orders").expect("the topic name should be valid"), + &observed( + &[1], + &[("retention.ms", "1000"), ("cleanup.policy", "compact")], + ), + ); + + topic_state.apply_config_changes( + &configs(&[("retention.ms", "2000"), ("max.message.bytes", "10")]), + &keys(&["cleanup.policy"]), + ); + + assert_eq!( + configs(&[("max.message.bytes", "10"), ("retention.ms", "2000")]), + topic_state.topic_configs + ); + } + + #[test] + fn report_a_different_replication_factor_as_unsupported() { + let observed = observed(&[3, 3], &[]); + + assert_eq!( + vec![UnsupportedChange::ChangeReplicationFactor { + desired: 2, + actual_min: 3, + actual_max: 3 + }], + unsupported_changes(&spec(None, Some(2), &[]), &observed) + ); + } + + #[test] + fn report_uneven_replica_counts_as_unsupported() { + let observed = observed(&[2, 3], &[]); + + assert_eq!( + vec![UnsupportedChange::ChangeReplicationFactor { + desired: 2, + actual_min: 2, + actual_max: 3 + }], + unsupported_changes(&spec(None, Some(2), &[]), &observed) + ); + } + + #[test] + fn accept_a_reassignment_towards_the_desired_replication_factor() { + let mut observed = observed(&[4, 2], &[]); + observed.reassignments = vec![PartitionReassignment { + index: 0, + removing_replicas: 2, + }]; + + assert_eq!( + Vec::::new(), + unsupported_changes(&spec(None, Some(2), &[]), &observed) + ); + } + + #[test] + fn report_the_target_replication_factor() { + let mut reassigning = observed(&[4, 2], &[]); + reassigning.reassignments = vec![PartitionReassignment { + index: 0, + removing_replicas: 2, + }]; + + assert_eq!(Some(2), replication_factor(&reassigning)); + assert_eq!(None, replication_factor(&observed(&[2, 3], &[]))); + } +} diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs new file mode 100644 index 000000000..14bc9b653 --- /dev/null +++ b/rust/agent-binary/src/topic/sync.rs @@ -0,0 +1,229 @@ +//! Brings the topic in Kafka to the state of its `KafkaTopic` spec. + +use std::collections::{BTreeMap, BTreeSet}; + +use snafu::ResultExt; +use stackable_kafka_admin::client::KafkaAdminClient; +use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopic}; +use stackable_operator::kube::ResourceExt; +use tracing::info; + +use super::{ + condition::TopicSyncOutcome, + controller::{Ctx, KafkaSnafu, Result}, + plan::{ObservedTopic, TopicPlan, TopicState, plan}, + validate::validate, +}; + +/// Sets `topic_state` to the topic in Kafka, if it exists and is managed by this `KafkaTopic`. +/// It is kept up to date with each applied change, also if a later step fails. +pub(crate) async fn sync( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + topic_state: &mut Option, +) -> Result { + // validate (no client required) + let topic_name = match validate(kafka_topic, &ctx.kafka_topics) { + Ok(topic_name) => topic_name, + Err(sync_outcome) => return Ok(sync_outcome), + }; + + // observe (Kafka) + let mut admin_client = ctx.admin_client.lock().await; + let observed = observe(&mut admin_client, &topic_name).await?; + *topic_state = observed + .as_ref() + .map(|observed| TopicState::observed(topic_name.clone(), observed)); + if kafka_topic.spec.topic_operation.reconciliation_paused { + return Ok(TopicSyncOutcome::ReconciliationPaused); + } + + // plan (no client required) + let topic_plan = plan(&kafka_topic.spec, observed.as_ref()); + let reassigning_partition_count = observed + .as_ref() + .map_or(0, |observed| observed.reassignments.len()); + + // execute (Kafka) + execute( + kafka_topic, + &mut admin_client, + topic_name, + &topic_plan, + topic_state, + ) + .await?; + + let sync_outcome = match topic_plan { + TopicPlan::Update { + unsupported_changes, + .. + } if !unsupported_changes.is_empty() => TopicSyncOutcome::UnsupportedChange { + changes: unsupported_changes, + }, + _ => TopicSyncOutcome::Synced { + reassigning_partition_count, + }, + }; + Ok(sync_outcome) +} + +async fn observe( + admin_client: &mut KafkaAdminClient, + topic_name: &TopicName, +) -> Result> { + let topic_name = topic_name.as_ref(); + let Some(topic) = admin_client + .describe_topic(topic_name) + .await + .context(KafkaSnafu)? + else { + return Ok(None); + }; + let partitions = topic + .partitions + .iter() + .map(|partition| partition.index) + .collect::>(); + Ok(Some(ObservedTopic { + topic_configs: admin_client + .describe_topic_configs(topic_name) + .await + .context(KafkaSnafu)?, + reassignments: admin_client + .list_partition_reassignments(topic_name, &partitions) + .await + .context(KafkaSnafu)?, + topic, + })) +} + +/// Applies the changes to `topic_state`, the topic in Kafka before them. +async fn execute( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: TopicName, + topic_plan: &TopicPlan, + topic_state: &mut Option, +) -> Result<()> { + match topic_plan { + TopicPlan::Create { + partition_count, + replication_factor, + config_overrides, + } => { + let created_state = create( + kafka_topic, + admin_client, + topic_name, + *partition_count, + *replication_factor, + config_overrides, + ) + .await?; + *topic_state = Some(created_state); + } + TopicPlan::Update { topic_update, .. } => { + // Partitions + if let Some(new_partition_count) = topic_update.new_partition_count { + grow_partitions(kafka_topic, admin_client, &topic_name, new_partition_count) + .await?; + if let Some(topic_state) = topic_state { + topic_state.partition_count = new_partition_count; + } + } + + // Config + if !topic_update.configs_to_set.is_empty() || !topic_update.configs_to_reset.is_empty() + { + alter_configs( + kafka_topic, + admin_client, + &topic_name, + &topic_update.configs_to_set, + &topic_update.configs_to_reset, + ) + .await?; + if let Some(topic_state) = topic_state { + topic_state.apply_config_changes( + &topic_update.configs_to_set, + &topic_update.configs_to_reset, + ); + } + } + } + } + Ok(()) +} + +/// The CreateTopics response contains the topic id, the counts and the effective configs, so no +/// describe is needed afterwards. Without the `DESCRIBE_CONFIGS` permission, Kafka creates the +/// topic but the request fails, and so does every later sync, because observing the topic needs +/// that permission as well. +async fn create( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: TopicName, + partition_count: Option, + replication_factor: Option, + config_overrides: &BTreeMap, +) -> Result { + let created = admin_client + .create_topic( + topic_name.as_ref(), + partition_count, + replication_factor, + config_overrides, + ) + .await + .context(KafkaSnafu)?; + info!( + topic = %topic_name, + partitions = created.partitions, + replication_factor = created.replication_factor, + configs = ?config_overrides, + kafka_topic = %kafka_topic.name_any(), + "Created the topic" + ); + Ok(TopicState::created(topic_name, created)) +} + +async fn grow_partitions( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: &TopicName, + partition_count: i32, +) -> Result<()> { + admin_client + .create_partitions(topic_name.as_ref(), partition_count) + .await + .context(KafkaSnafu)?; + info!( + topic = %topic_name, + partitions = partition_count, + kafka_topic = %kafka_topic.name_any(), + "Increased the number of partitions" + ); + Ok(()) +} + +async fn alter_configs( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: &TopicName, + configs_to_set: &BTreeMap, + configs_to_reset: &BTreeSet, +) -> Result<()> { + admin_client + .incremental_alter_configs(topic_name.as_ref(), configs_to_set, configs_to_reset) + .await + .context(KafkaSnafu)?; + info!( + topic = %topic_name, + set = ?configs_to_set, + reset = ?configs_to_reset, + kafka_topic = %kafka_topic.name_any(), + "Changed the topic configs" + ); + Ok(()) +} diff --git a/rust/agent-binary/src/topic/test_support.rs b/rust/agent-binary/src/topic/test_support.rs new file mode 100644 index 000000000..bba58c0ba --- /dev/null +++ b/rust/agent-binary/src/topic/test_support.rs @@ -0,0 +1,28 @@ +use std::str::FromStr; + +use stackable_kafka_shared::crd::topic::{ + TopicName, + v1alpha1::{KafkaClusterRef, KafkaTopic, KafkaTopicSpec}, +}; +use stackable_operator::v2::types::operator::ClusterName; + +/// A `KafkaTopic` of the cluster `simple-kafka` with one partition, one replica and no configs. +pub fn kafka_topic(name: &str, topic_name: Option<&str>) -> KafkaTopic { + KafkaTopic::new( + name, + KafkaTopicSpec { + cluster_ref: KafkaClusterRef { + name: ClusterName::from_str("simple-kafka") + .expect("the cluster name should be valid"), + }, + topic_name: topic_name.map(|topic_name| { + TopicName::from_str(topic_name).expect("the topic name should be valid") + }), + partitions: Some(1), + replication_factor: Some(1), + config_overrides: Default::default(), + deletion_policy: Default::default(), + topic_operation: Default::default(), + }, + ) +} diff --git a/rust/agent-binary/src/topic/update_status.rs b/rust/agent-binary/src/topic/update_status.rs new file mode 100644 index 000000000..b7fdcfae6 --- /dev/null +++ b/rust/agent-binary/src/topic/update_status.rs @@ -0,0 +1,174 @@ +//! The status of a `KafkaTopic` for the outcome of a sync. + +use snafu::ResultExt; +use stackable_kafka_shared::crd::topic::v1alpha1::{KafkaTopic, KafkaTopicStatus}; +use stackable_operator::{ + agent::condition::update_conditions, + k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, +}; + +use super::{ + condition::{TopicSyncOutcome, conditions_from}, + controller::{Ctx, Result, UpdateStatusSnafu}, + plan::TopicState, +}; +use crate::AGENT_NAME; + +/// Patches the status only if it changed, so the agent does not trigger its own watch. +pub(crate) async fn update_status( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> Result<()> { + let current_status = kafka_topic.status.clone().unwrap_or_default(); + let new_status = status_from( + ¤t_status, + sync_outcome, + topic_state, + &Time(Timestamp::now()), + kafka_topic.metadata.generation, + ); + + if new_status != current_status { + ctx.client + .apply_patch_status(AGENT_NAME, kafka_topic, &new_status) + .await + .context(UpdateStatusSnafu)?; + } + Ok(()) +} + +/// Fields not known after the outcome are kept from `current_status`. +fn status_from( + current_status: &KafkaTopicStatus, + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, + now: &Time, + observed_generation: Option, +) -> KafkaTopicStatus { + let conditions = update_conditions( + ¤t_status.conditions, + conditions_from(sync_outcome, topic_state), + now, + observed_generation, + ); + match (sync_outcome, topic_state) { + (_, Some(topic_state)) => KafkaTopicStatus { + conditions, + topic_id: Some(topic_state.id.clone()), + topic_name: Some(topic_state.name.clone()), + partitions: Some(topic_state.partition_count), + replication_factor: topic_state.replication_factor, + config: topic_state.topic_configs.clone(), + }, + // The topic does not exist or is managed by another KafkaTopic. + (TopicSyncOutcome::Conflict { .. } | TopicSyncOutcome::ReconciliationPaused, None) => { + KafkaTopicStatus { + conditions, + ..KafkaTopicStatus::default() + } + } + _ => KafkaTopicStatus { + conditions, + ..current_status.clone() + }, + } +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, str::FromStr}; + + use stackable_kafka_shared::crd::topic::TopicName; + + use super::*; + + fn now() -> Time { + Time(Timestamp::from_second(10).expect("the timestamp should be valid")) + } + + fn topic_state() -> TopicState { + TopicState { + name: TopicName::from_str("orders").expect("the topic name should be valid"), + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partition_count: 6, + replication_factor: Some(3), + topic_configs: BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]), + } + } + + fn synced() -> TopicSyncOutcome { + TopicSyncOutcome::Synced { + reassigning_partition_count: 0, + } + } + + #[test] + fn show_the_topic_in_kafka() { + let status = status_from( + &KafkaTopicStatus::default(), + &synced(), + Some(&topic_state()), + &now(), + Some(2), + ); + + assert_eq!(Some("UXyU9i5ARn6W00ON2taeWA".to_owned()), status.topic_id); + assert_eq!(Some(6), status.partitions); + assert_eq!(Some(3), status.replication_factor); + assert_eq!( + Some("1000"), + status.config.get("retention.ms").map(String::as_str) + ); + assert!( + status + .conditions + .iter() + .all(|condition| condition.observed_generation == Some(2)) + ); + } + + #[test] + fn keep_the_topic_if_kafka_cannot_be_reached() { + let current = status_from( + &KafkaTopicStatus::default(), + &synced(), + Some(&topic_state()), + &now(), + Some(1), + ); + + let status = status_from( + ¤t, + &TopicSyncOutcome::ConnectionFailed { + message: "timed out".to_owned(), + }, + None, + &now(), + Some(1), + ); + + assert_eq!(current.topic_id, status.topic_id); + assert_eq!(current.config, status.config); + } + + #[rstest::rstest] + #[case::conflict(TopicSyncOutcome::Conflict { message: String::new() })] + #[case::paused_without_topic(TopicSyncOutcome::ReconciliationPaused)] + fn clear_the_topic_if_it_is_not_managed(#[case] sync_outcome: TopicSyncOutcome) { + let current = status_from( + &KafkaTopicStatus::default(), + &synced(), + Some(&topic_state()), + &now(), + Some(1), + ); + + let status = status_from(¤t, &sync_outcome, None, &now(), Some(1)); + + assert_eq!(None, status.topic_id); + assert!(status.config.is_empty()); + assert_eq!(2, status.conditions.len()); + } +} diff --git a/rust/agent-binary/src/topic/validate.rs b/rust/agent-binary/src/topic/validate.rs new file mode 100644 index 000000000..d5055d4dc --- /dev/null +++ b/rust/agent-binary/src/topic/validate.rs @@ -0,0 +1,234 @@ +//! Whether a `KafkaTopic` can be synced, without talking to Kafka. + +use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopic}; +use stackable_operator::kube::{ResourceExt, core::DeserializeGuard, runtime::reflector::Store}; + +use super::{condition::TopicSyncOutcome, plan::THROTTLE_CONFIG_KEYS}; + +/// Returns the name of the topic in Kafka, or why the `KafkaTopic` cannot be synced. +pub(crate) fn validate( + kafka_topic: &KafkaTopic, + kafka_topics: &Store>, +) -> Result { + let topic_name = kafka_topic + .topic_name() + .map_err(|error| TopicSyncOutcome::InvalidSpec { + message: format!( + "metadata.name is not a valid topic name, recreate the KafkaTopic with topicName \ + set: {error}" + ), + })?; + + let throttle_keys = THROTTLE_CONFIG_KEYS + .iter() + .filter(|key| kafka_topic.spec.config_overrides.contains_key(**key)) + .copied() + .collect::>(); + if !throttle_keys.is_empty() { + return Err(TopicSyncOutcome::InvalidSpec { + message: format!( + "the config keys {throttle_keys:?} are managed by partition reassignments and \ + cannot be set" + ), + }); + } + + if let Some(owner) = conflicting_owner(kafka_topic, &topic_name, kafka_topics) { + return Err(TopicSyncOutcome::Conflict { + message: format!("the topic {topic_name} is managed by the KafkaTopic {owner:?}"), + }); + } + + Ok(topic_name) +} + +/// The oldest other `KafkaTopic` for the same topic of the same cluster, by creation time and +/// name. Checked against all `KafkaTopic`s in the store, so the result does not depend on the +/// reconcile order. +pub(crate) fn conflicting_owner( + kafka_topic: &KafkaTopic, + topic_name: &TopicName, + kafka_topics: &Store>, +) -> Option { + let key = |kafka_topic: &KafkaTopic| { + ( + kafka_topic.metadata.creation_timestamp.clone(), + kafka_topic.name_any(), + ) + }; + kafka_topics + .state() + .iter() + .filter_map(|other_kafka_topic| other_kafka_topic.0.as_ref().ok()) + .filter(|other_kafka_topic| is_same_topic(kafka_topic, other_kafka_topic, topic_name)) + .min_by_key(|other_kafka_topic| key(other_kafka_topic)) + .filter(|oldest_kafka_topic| key(oldest_kafka_topic) != key(kafka_topic)) + .map(ResourceExt::name_any) +} + +/// Whether another `KafkaTopic` that is not being deleted declares the same topic of the same +/// cluster, regardless of its age. +pub(crate) fn has_other_kafka_topic( + kafka_topic: &KafkaTopic, + topic_name: &TopicName, + kafka_topics: &Store>, +) -> bool { + kafka_topics + .state() + .iter() + .filter_map(|other_kafka_topic| other_kafka_topic.0.as_ref().ok()) + .any(|other_kafka_topic| { + other_kafka_topic.name_any() != kafka_topic.name_any() + && other_kafka_topic.metadata.deletion_timestamp.is_none() + && is_same_topic(kafka_topic, other_kafka_topic, topic_name) + }) +} + +fn is_same_topic( + kafka_topic: &KafkaTopic, + other_kafka_topic: &KafkaTopic, + topic_name: &TopicName, +) -> bool { + other_kafka_topic.spec.cluster_ref.name == kafka_topic.spec.cluster_ref.name + && other_kafka_topic + .topic_name() + .is_ok_and(|other_topic_name| &other_topic_name == topic_name) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use stackable_operator::{ + k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, + kube::runtime::watcher, + }; + + use super::*; + use crate::topic::test_support::kafka_topic; + + fn created_topic( + name: &str, + created_at: i64, + topic_name: Option<&str>, + ) -> DeserializeGuard { + let mut topic = kafka_topic(name, topic_name); + topic.metadata.creation_timestamp = Some(Time( + Timestamp::from_second(created_at).expect("the timestamp should be valid"), + )); + DeserializeGuard(Ok(topic)) + } + + fn store(topics: Vec>) -> Store> { + let (store, mut writer) = stackable_operator::kube::runtime::reflector::store(); + for topic in topics { + writer.apply_watcher_event(&watcher::Event::Apply(topic)); + } + store + } + + fn inner(topic: &DeserializeGuard) -> &KafkaTopic { + topic.0.as_ref().expect("the topic should be valid") + } + + fn topic_name(name: &str) -> TopicName { + TopicName::from_str(name).expect("the topic name should be valid") + } + + #[test] + fn the_oldest_kafka_topic_owns_the_topic() { + let older = created_topic("older", 10, Some("orders")); + let newer = created_topic("newer", 20, Some("orders")); + let other_topic = created_topic("other", 5, Some("payments")); + let topics = store(vec![older.clone(), newer.clone(), other_topic]); + + assert_eq!( + None, + conflicting_owner(inner(&older), &topic_name("orders"), &topics) + ); + assert_eq!( + Some("older".to_owned()), + conflicting_owner(inner(&newer), &topic_name("orders"), &topics) + ); + } + + #[test] + fn the_name_decides_between_kafka_topics_of_the_same_age() { + let first = created_topic("a", 10, Some("orders")); + let second = created_topic("b", 10, Some("orders")); + let topics = store(vec![second.clone(), first.clone()]); + + assert_eq!( + None, + conflicting_owner(inner(&first), &topic_name("orders"), &topics) + ); + assert_eq!( + Some("a".to_owned()), + conflicting_owner(inner(&second), &topic_name("orders"), &topics) + ); + } + + #[test] + fn compare_the_effective_topic_names() { + let explicit = created_topic("explicit", 10, Some("orders")); + let from_metadata_name = created_topic("orders", 20, None); + let topics = store(vec![explicit.clone(), from_metadata_name.clone()]); + + assert_eq!( + Some("explicit".to_owned()), + conflicting_owner(inner(&from_metadata_name), &topic_name("orders"), &topics) + ); + } + + #[test] + fn find_other_kafka_topics_regardless_of_their_age() { + let older = created_topic("older", 10, Some("orders")); + let newer = created_topic("newer", 20, Some("orders")); + let mut deleted = created_topic("deleted", 30, Some("payments")); + if let Ok(deleted) = &mut deleted.0 { + deleted.metadata.deletion_timestamp = deleted.metadata.creation_timestamp.clone(); + } + let payments = created_topic("payments", 40, None); + let topics = store(vec![older.clone(), newer, deleted, payments.clone()]); + + assert!(has_other_kafka_topic( + inner(&older), + &topic_name("orders"), + &topics + )); + assert!(!has_other_kafka_topic( + inner(&payments), + &topic_name("payments"), + &topics + )); + } + + #[rstest::rstest] + #[case::no_throttle(&[("retention.ms", "1000")], false)] + #[case::throttle(&[("follower.replication.throttled.replicas", "*")], true)] + fn reject_throttle_config_keys(#[case] config: &[(&str, &str)], #[case] invalid: bool) { + let mut topic = kafka_topic("orders", Some("orders")); + topic.spec.config_overrides = config + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + + assert_eq!( + invalid, + matches!( + validate(&topic, &store(vec![])), + Err(TopicSyncOutcome::InvalidSpec { .. }) + ) + ); + } + + #[test] + fn reject_a_too_long_metadata_name_without_topic_name() { + let topic = kafka_topic(&"a".repeat(250), None); + + assert!(matches!( + validate(&topic, &store(vec![])), + Err(TopicSyncOutcome::InvalidSpec { .. }) + )); + } +} diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 1c349de28..dcb5f74ad 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -95,6 +95,14 @@ pub fn all_agents_selector() -> Selector { Selector::from_iter(BTreeMap::from(label::all_agents_selector(&PRODUCT_NAME))) } +/// Type-safe names for the resources of the agent of the given cluster. +pub fn agent_resource_names_of(cluster_name: &ClusterName) -> AgentResourceNames { + AgentResourceNames { + cluster_name: cluster_name.clone(), + product_name: PRODUCT_NAME.clone(), + } +} + #[derive(Snafu, Debug)] pub enum PodDescriptorsError { #[snafu(display( @@ -300,10 +308,7 @@ impl ValidatedCluster { /// Type-safe names for the resources of the agent. pub fn agent_resource_names(&self) -> AgentResourceNames { - AgentResourceNames { - cluster_name: self.name.clone(), - product_name: PRODUCT_NAME.clone(), - } + agent_resource_names_of(&self.name) } /// Type-safe names for the resources of a given role group. diff --git a/rust/operator-binary/src/controller/build/properties/listener.rs b/rust/operator-binary/src/controller/build/properties/listener.rs index 2cb533769..e7a460b7c 100644 --- a/rust/operator-binary/src/controller/build/properties/listener.rs +++ b/rust/operator-binary/src/controller/build/properties/listener.rs @@ -157,8 +157,6 @@ pub(crate) fn pod_fqdn( format!("${{env:POD_NAME}}.{sts_service_name}.{namespace}.svc.{cluster_domain}") } -/// `..svc.:`, which resolves to the Pods of a headless -/// Service. pub(crate) fn headless_service_address( headless_service_name: &ServiceName, namespace: &NamespaceName, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 7a0a9c5f9..7df6ab26e 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -1,12 +1,15 @@ // TODO: Look into how to properly resolve `clippy::large_enum_variant`. // This will need changes in our and upstream error types. -#![allow(clippy::result_large_err)] +#![allow(clippy::large_enum_variant, clippy::result_large_err)] use std::sync::Arc; use anyhow::anyhow; use clap::Parser; use futures::{FutureExt, StreamExt, TryFutureExt}; +use stackable_kafka_shared::crd::topic::{ + KafkaTopic, KafkaTopicVersion, v1alpha1 as topic_v1alpha1, +}; use stackable_operator::{ YamlSchema, cli::{Command, RunArguments}, @@ -39,11 +42,13 @@ use stackable_operator::{ use crate::{ controller::{KAFKA_FULL_CONTROLLER_NAME, all_agents_selector}, crd::{KAFKA_OPERATOR_NAME, KafkaCluster, KafkaClusterVersion, v1alpha1}, + topic_controller::{TOPIC_FULL_CONTROLLER_NAME, kafka_topics_of_cluster}, webhooks::conversion::create_webhook_server, }; mod controller; mod crd; +mod topic_controller; mod webhooks; mod built_info { @@ -79,8 +84,12 @@ struct KafkaRun { async fn main() -> anyhow::Result<()> { let opts = Opts::parse(); match opts.cmd { - Command::Crd => KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1)? - .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?, + Command::Crd => { + KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1)? + .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; + KafkaTopic::merged_crd(KafkaTopicVersion::V1Alpha1)? + .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; + } Command::Run(KafkaRun { common: RunArguments { @@ -128,6 +137,10 @@ async fn main() -> anyhow::Result<()> { "CRD {crd} established", crd = v1alpha1::KafkaCluster::crd_name() )); + let kafka_topic_crd_check = readiness_checks.register(format!( + "CRD {crd} established", + crd = topic_v1alpha1::KafkaTopic::crd_name() + )); let webhook_server = create_webhook_server( &operator_environment, @@ -229,13 +242,86 @@ async fn main() -> anyhow::Result<()> { ) .map(anyhow::Ok); + // Topic controller to clean up after deleted or unavailable agents + let topic_event_recorder = Arc::new(Recorder::new( + client.as_kube_client(), + Reporter { + controller: TOPIC_FULL_CONTROLLER_NAME.to_string(), + instance: None, + }, + )); + + let topic_controller = Controller::new( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + ); + let kafka_topic_store_for_clusters = topic_controller.store(); + let kafka_topic_store_for_agents = topic_controller.store(); + let topic_controller = topic_controller + .watches( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + move |kafka_cluster| { + kafka_topics_of_cluster( + &kafka_topic_store_for_clusters, + kafka_cluster.namespace(), + Some(&kafka_cluster.name_any()), + ) + }, + ) + .watches( + watch_namespace.get_api::>(&client), + watcher::Config::default().labels_from(&all_agents_selector()), + move |agent_deployment| { + kafka_topics_of_cluster( + &kafka_topic_store_for_agents, + agent_deployment.namespace(), + agent_deployment + .labels() + .get("app.kubernetes.io/instance") + .map(String::as_str), + ) + }, + ) + .graceful_shutdown_on(sigterm_watcher.handle()) + .run( + topic_controller::reconcile_topic, + topic_controller::error_policy, + Arc::new(topic_controller::Ctx { + client: client.clone(), + }), + ) + .for_each_concurrent(16, move |result| { + let topic_event_recorder = topic_event_recorder.clone(); + async move { + report_controller_reconciled( + &topic_event_recorder, + TOPIC_FULL_CONTROLLER_NAME, + &result, + ) + .await; + } + }) + .map(anyhow::Ok); + let delayed_kafka_controller = async { signal::crd_established(&client, v1alpha1::KafkaCluster::crd_name()).await?; kafka_cluster_crd_check.mark_passed(); kafka_controller.await }; - futures::try_join!(delayed_kafka_controller, eos_checker, webhook_server)?; + let delayed_topic_controller = async { + signal::crd_established(&client, topic_v1alpha1::KafkaTopic::crd_name()).await?; + kafka_topic_crd_check.mark_passed(); + topic_controller.await + }; + + futures::try_join!( + delayed_kafka_controller, + delayed_topic_controller, + eos_checker, + webhook_server + )?; } }; diff --git a/rust/operator-binary/src/topic_controller.rs b/rust/operator-binary/src/topic_controller.rs new file mode 100644 index 000000000..11aa540b5 --- /dev/null +++ b/rust/operator-binary/src/topic_controller.rs @@ -0,0 +1,538 @@ +//! Takes over `KafkaTopic`s while no agent is available: reports it in their status and removes +//! their finalizer once the KafkaCluster is gone, so deleting the KafkaCluster or the namespace +//! does not block. + +use std::sync::Arc; + +use const_format::concatcp; +use serde_json::json; +use snafu::{ResultExt, Snafu}; +use stackable_kafka_shared::crd::topic::{KAFKA_TOPIC_FINALIZER, v1alpha1::KafkaTopic}; +use stackable_operator::{ + agent::condition::{ + ConditionStatus, ManagedCondition, ManagedConditionType, update_conditions, + }, + k8s_openapi::{ + api::apps::v1::Deployment, + apimachinery::pkg::apis::meta::v1::Time, + jiff::{SignedDuration, Timestamp}, + }, + kube::{ + ResourceExt, + core::{DeserializeGuard, error_boundary}, + runtime::{ + controller::Action, + reflector::{ObjectRef, Store}, + }, + }, + logging::controller::ReconcilerError, + shared::time::Duration, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::{ + controller::agent_resource_names_of, + crd::{KAFKA_OPERATOR_NAME, v1alpha1::KafkaCluster}, +}; + +pub const TOPIC_CONTROLLER_NAME: &str = "kafkatopic"; +pub const TOPIC_FULL_CONTROLLER_NAME: &str = + concatcp!(TOPIC_CONTROLLER_NAME, '.', KAFKA_OPERATOR_NAME); + +const AGENT_UNAVAILABLE_REASON: &str = "AgentUnavailable"; + +/// `DeploymentConditionType` of `apps/v1`, true once the minimum number of replicas is available +const DEPLOYMENT_CONDITION_AVAILABLE: &str = "Available"; + +/// Shorter outages of the agent are not reported, such as a rollout that pulls a new image. +const AGENT_UNAVAILABLE_GRACE_PERIOD: SignedDuration = SignedDuration::from_secs(60); + +#[derive(Debug, Snafu, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("the KafkaTopic is invalid"))] + InvalidKafkaTopic { + source: error_boundary::InvalidObject, + }, + + #[snafu(display("failed to get the KafkaCluster"))] + GetKafkaCluster { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to get the agent Deployment"))] + GetAgentDeployment { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to update the status of the KafkaTopic"))] + UpdateStatus { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to remove the finalizer of the KafkaTopic"))] + RemoveFinalizer { + source: stackable_operator::client::Error, + }, +} + +type Result = std::result::Result; + +impl ReconcilerError for Error { + fn category(&self) -> &'static str { + ErrorDiscriminants::from(self).into() + } +} + +pub struct Ctx { + pub client: stackable_operator::client::Client, +} + +#[derive(Debug, Eq, PartialEq)] +enum ClusterState { + /// Does not exist or has a `deletionTimestamp` + MissingOrDeleting, + ManagementAccessDisabled, + ManagementAccessEnabled, +} + +#[derive(Debug, Eq, PartialEq)] +enum AgentState { + Available, + /// `None` if the agent Deployment does not exist or has no replicas + UnavailableSince(Option), +} + +#[derive(Debug, Eq, PartialEq)] +enum OperatorAction { + LeaveToAgent, + WaitForAgent { remaining: std::time::Duration }, + RemoveFinalizer, + SetAgentUnavailableConditions { message: &'static str }, +} + +pub async fn reconcile_topic( + kafka_topic: Arc>, + ctx: Arc, +) -> Result { + let kafka_topic = kafka_topic + .0 + .as_ref() + .map_err(error_boundary::InvalidObject::clone) + .context(InvalidKafkaTopicSnafu)?; + let namespace = kafka_topic.namespace().unwrap_or_default(); + let cluster_name = &kafka_topic.spec.cluster_ref.name; + + let kafka_cluster = ctx + .client + .get_opt::>(cluster_name.as_ref(), &namespace) + .await + .context(GetKafkaClusterSnafu)?; + let agent_deployment_name = agent_resource_names_of(cluster_name).deployment_name(); + let agent_deployment = ctx + .client + .get_opt::(agent_deployment_name.as_ref(), &namespace) + .await + .context(GetAgentDeploymentSnafu)?; + + match operator_action_for( + kafka_topic, + &cluster_state_from(kafka_cluster.as_ref()), + &agent_state_from(agent_deployment.as_ref()), + Timestamp::now(), + ) { + OperatorAction::LeaveToAgent => {} + OperatorAction::WaitForAgent { remaining } => return Ok(Action::requeue(remaining)), + OperatorAction::RemoveFinalizer => remove_finalizer(kafka_topic, &ctx).await?, + OperatorAction::SetAgentUnavailableConditions { message } => { + set_agent_unavailable_conditions(kafka_topic, &ctx, message).await? + } + } + Ok(Action::await_change()) +} + +pub fn error_policy( + _kafka_topic: Arc>, + error: &Error, + _ctx: Arc, +) -> Action { + match error { + Error::InvalidKafkaTopic { .. } => Action::await_change(), + _ => Action::requeue(*Duration::from_secs(30)), + } +} + +pub fn kafka_topics_of_cluster( + kafka_topics: &Store>, + namespace: Option, + cluster_name: Option<&str>, +) -> Vec>> { + kafka_topics + .state() + .into_iter() + .filter(|kafka_topic| { + kafka_topic.0.as_ref().is_ok_and(|kafka_topic| { + kafka_topic.namespace() == namespace + && Some(kafka_topic.spec.cluster_ref.name.as_ref()) == cluster_name + }) + }) + .map(|kafka_topic| ObjectRef::from_obj(&*kafka_topic)) + .collect() +} + +fn cluster_state_from(kafka_cluster: Option<&DeserializeGuard>) -> ClusterState { + match kafka_cluster.map(|kafka_cluster| &kafka_cluster.0) { + None => ClusterState::MissingOrDeleting, + Some(Ok(kafka_cluster)) if kafka_cluster.metadata.deletion_timestamp.is_some() => { + ClusterState::MissingOrDeleting + } + Some(Ok(kafka_cluster)) + if !kafka_cluster + .spec + .cluster_config + .management_access + .as_ref() + .is_some_and(|management_access| management_access.enabled) => + { + ClusterState::ManagementAccessDisabled + } + // An invalid KafkaCluster keeps its agent, so the state of the agent decides. + Some(_) => ClusterState::ManagementAccessEnabled, + } +} + +fn operator_action_for( + kafka_topic: &KafkaTopic, + cluster_state: &ClusterState, + agent_state: &AgentState, + now: Timestamp, +) -> OperatorAction { + match cluster_state { + ClusterState::MissingOrDeleting if kafka_topic.metadata.deletion_timestamp.is_some() => { + if kafka_topic + .finalizers() + .iter() + .any(|finalizer| finalizer == KAFKA_TOPIC_FINALIZER) + { + OperatorAction::RemoveFinalizer + } else { + OperatorAction::LeaveToAgent + } + } + ClusterState::MissingOrDeleting => OperatorAction::SetAgentUnavailableConditions { + message: "the KafkaCluster does not exist", + }, + ClusterState::ManagementAccessDisabled => OperatorAction::SetAgentUnavailableConditions { + message: "the KafkaCluster has no agent, because clusterConfig.managementAccess is \ + not enabled", + }, + ClusterState::ManagementAccessEnabled => match agent_state { + AgentState::Available => OperatorAction::LeaveToAgent, + AgentState::UnavailableSince(Some(since)) + if now.duration_since(*since) < AGENT_UNAVAILABLE_GRACE_PERIOD => + { + OperatorAction::WaitForAgent { + remaining: (AGENT_UNAVAILABLE_GRACE_PERIOD - now.duration_since(*since)) + .unsigned_abs(), + } + } + AgentState::UnavailableSince(_) => OperatorAction::SetAgentUnavailableConditions { + message: "the agent of the KafkaCluster is not available", + }, + }, + } +} + +/// A Deployment that has no `Available` condition yet is unavailable since its creation. One with +/// zero replicas, e.g. of a stopped KafkaCluster, is unavailable at once, although Kubernetes +/// reports it as `Available`. +fn agent_state_from(agent_deployment: Option<&Deployment>) -> AgentState { + let Some(agent_deployment) = agent_deployment else { + return AgentState::UnavailableSince(None); + }; + if agent_deployment + .spec + .as_ref() + .and_then(|spec| spec.replicas) + == Some(0) + { + return AgentState::UnavailableSince(None); + } + let available_condition = agent_deployment + .status + .as_ref() + .and_then(|status| status.conditions.as_ref()) + .and_then(|conditions| { + conditions + .iter() + .find(|condition| condition.type_ == DEPLOYMENT_CONDITION_AVAILABLE) + }); + match available_condition { + Some(condition) if condition.status == "True" => AgentState::Available, + Some(condition) => { + AgentState::UnavailableSince(condition.last_transition_time.as_ref().map(|time| time.0)) + } + None => AgentState::UnavailableSince( + agent_deployment + .metadata + .creation_timestamp + .as_ref() + .map(|time| time.0), + ), + } +} + +/// Only the conditions are applied, so the other fields stay owned by the agent and keep the +/// last state it observed. +async fn set_agent_unavailable_conditions( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + message: &str, +) -> Result<()> { + let current_conditions = kafka_topic + .status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or_default(); + let new_conditions = update_conditions( + current_conditions, + vec![ + agent_unavailable_condition( + ManagedConditionType::Ready, + ConditionStatus::Unknown, + message, + ), + agent_unavailable_condition( + ManagedConditionType::Synced, + ConditionStatus::False, + message, + ), + ], + &Time(Timestamp::now()), + kafka_topic.metadata.generation, + ); + + if new_conditions != current_conditions { + ctx.client + .apply_patch_status( + TOPIC_CONTROLLER_NAME, + kafka_topic, + &json!({ "conditions": new_conditions }), + ) + .await + .context(UpdateStatusSnafu)?; + } + Ok(()) +} + +fn agent_unavailable_condition( + type_: ManagedConditionType, + status: ConditionStatus, + message: &str, +) -> ManagedCondition { + ManagedCondition { + type_, + status, + reason: AGENT_UNAVAILABLE_REASON.to_owned(), + message: message.to_owned(), + } +} + +/// Fails if the `KafkaTopic` was changed since it was read, so concurrent changes to the +/// finalizers, e.g. by an agent that is still shutting down, are not lost. +async fn remove_finalizer(kafka_topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + let finalizers = kafka_topic + .finalizers() + .iter() + .filter(|finalizer| *finalizer != KAFKA_TOPIC_FINALIZER) + .cloned() + .collect::>(); + let patch = json!({ + "metadata": { + "resourceVersion": kafka_topic.metadata.resource_version, + "finalizers": finalizers, + } + }); + ctx.client + .merge_patch(kafka_topic, patch) + .await + .context(RemoveFinalizerSnafu)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use stackable_operator::{k8s_openapi::api::apps::v1::DeploymentSpec, kube::api::ObjectMeta}; + + use super::*; + + fn kafka_topic(deleting: bool, with_finalizer: bool) -> KafkaTopic { + let kafka_topic: KafkaTopic = serde_json::from_value(json!({ + "apiVersion": "kafka.stackable.tech/v1alpha1", + "kind": "KafkaTopic", + "metadata": { "name": "orders", "namespace": "default" }, + "spec": { "clusterRef": { "name": "simple-kafka" } }, + })) + .expect("the KafkaTopic should be valid"); + KafkaTopic { + metadata: ObjectMeta { + deletion_timestamp: deleting.then_some(Time(Timestamp::UNIX_EPOCH)), + finalizers: with_finalizer.then(|| vec![KAFKA_TOPIC_FINALIZER.to_owned()]), + ..kafka_topic.metadata + }, + ..kafka_topic + } + } + + fn now() -> Timestamp { + Timestamp::from_second(1_000).expect("the timestamp should be valid") + } + + fn seconds_ago(seconds: i64) -> Timestamp { + now() - SignedDuration::from_secs(seconds) + } + + #[rstest] + #[case::cluster_missing(ClusterState::MissingOrDeleting, AgentState::Available, true)] + #[case::cluster_missing_without_agent( + ClusterState::MissingOrDeleting, + AgentState::UnavailableSince(None), + true + )] + #[case::management_access_disabled( + ClusterState::ManagementAccessDisabled, + AgentState::UnavailableSince(None), + false + )] + #[case::agent_available(ClusterState::ManagementAccessEnabled, AgentState::Available, false)] + #[case::agent_unavailable( + ClusterState::ManagementAccessEnabled, + AgentState::UnavailableSince(None), + false + )] + fn remove_the_finalizer_only_if_the_cluster_is_missing_or_deleting( + #[case] cluster_state: ClusterState, + #[case] agent_state: AgentState, + #[case] removes_finalizer: bool, + ) { + assert_eq!( + removes_finalizer, + operator_action_for( + &kafka_topic(true, true), + &cluster_state, + &agent_state, + now() + ) == OperatorAction::RemoveFinalizer + ); + } + + #[rstest] + #[case::cluster_missing(ClusterState::MissingOrDeleting, AgentState::Available, true)] + #[case::management_access_disabled( + ClusterState::ManagementAccessDisabled, + AgentState::Available, + true + )] + #[case::agent_available(ClusterState::ManagementAccessEnabled, AgentState::Available, false)] + #[case::agent_deployment_missing( + ClusterState::ManagementAccessEnabled, + AgentState::UnavailableSince(None), + true + )] + #[case::agent_unavailable_for_longer_than_the_grace_period( + ClusterState::ManagementAccessEnabled, + AgentState::UnavailableSince(Some(seconds_ago(61))), + true + )] + fn set_agent_unavailable_conditions_without_an_available_agent( + #[case] cluster_state: ClusterState, + #[case] agent_state: AgentState, + #[case] sets_conditions: bool, + ) { + assert_eq!( + sets_conditions, + matches!( + operator_action_for( + &kafka_topic(false, false), + &cluster_state, + &agent_state, + now() + ), + OperatorAction::SetAgentUnavailableConditions { .. } + ) + ); + } + + #[test] + fn wait_for_an_agent_that_is_unavailable_within_the_grace_period() { + assert_eq!( + OperatorAction::WaitForAgent { + remaining: std::time::Duration::from_secs(50) + }, + operator_action_for( + &kafka_topic(false, false), + &ClusterState::ManagementAccessEnabled, + &AgentState::UnavailableSince(Some(seconds_ago(10))), + now() + ) + ); + } + + #[test] + fn leave_a_deleting_kafka_topic_without_finalizer_alone() { + assert_eq!( + OperatorAction::LeaveToAgent, + operator_action_for( + &kafka_topic(true, false), + &ClusterState::MissingOrDeleting, + &AgentState::UnavailableSince(None), + now() + ) + ); + } + + fn agent_deployment(available: Option<&str>) -> Deployment { + serde_json::from_value(json!({ + "metadata": { "creationTimestamp": "1970-01-01T00:10:00Z" }, + "status": { + "conditions": available.map(|status| vec![json!({ + "type": "Available", + "status": status, + "lastTransitionTime": "1970-01-01T00:15:00Z", + })]), + }, + })) + .expect("the Deployment should be valid") + } + + fn scaled_to_zero(mut agent_deployment: Deployment) -> Deployment { + agent_deployment.spec = Some(DeploymentSpec { + replicas: Some(0), + ..DeploymentSpec::default() + }); + agent_deployment + } + + #[rstest] + #[case::missing(None, AgentState::UnavailableSince(None))] + #[case::available(Some(agent_deployment(Some("True"))), AgentState::Available)] + #[case::scaled_to_zero( + Some(scaled_to_zero(agent_deployment(Some("True")))), + AgentState::UnavailableSince(None) + )] + #[case::unavailable_since_the_transition( + Some(agent_deployment(Some("False"))), + AgentState::UnavailableSince(Some(Timestamp::from_second(900).expect("valid"))) + )] + #[case::unavailable_since_the_creation( + Some(agent_deployment(None)), + AgentState::UnavailableSince(Some(Timestamp::from_second(600).expect("valid"))) + )] + fn derive_the_agent_state_from_the_deployment( + #[case] agent_deployment: Option, + #[case] expected: AgentState, + ) { + assert_eq!(expected, agent_state_from(agent_deployment.as_ref())); + } +} diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index f8921cf9e..ffe0115ad 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -1,4 +1,5 @@ use snafu::{ResultExt, Snafu}; +use stackable_kafka_shared::crd::topic::{KafkaTopic, KafkaTopicVersion}; use stackable_operator::{ cli::OperatorEnvironmentOptions, kube::{Client, core::crd::MergeError}, @@ -29,10 +30,16 @@ pub async fn create_webhook_server( readiness_checks: HealthCheckRegistry, client: Client, ) -> Result { - let crds_and_handlers = vec![( - KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1).context(MergeCrdSnafu)?, - KafkaCluster::try_convert, - )]; + let crds_and_handlers = vec![ + ( + KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1).context(MergeCrdSnafu)?, + KafkaCluster::try_convert as fn(_) -> _, + ), + ( + KafkaTopic::merged_crd(KafkaTopicVersion::V1Alpha1).context(MergeCrdSnafu)?, + KafkaTopic::try_convert as fn(_) -> _, + ), + ]; let conversion_webhook_options = ConversionWebhookOptions { disable_crd_maintenance, diff --git a/rust/shared/Cargo.toml b/rust/shared/Cargo.toml index a26223f1c..0e51994c1 100644 --- a/rust/shared/Cargo.toml +++ b/rust/shared/Cargo.toml @@ -11,7 +11,16 @@ publish = false [dependencies] stackable-operator.workspace = true +regex.workspace = true +serde.workspace = true +serde_json.workspace = true +snafu.workspace = true strum.workspace = true [dev-dependencies] +indoc.workspace = true rstest.workspace = true + +[features] +# Referenced by the attributed_string_type! macro of stackable-operator +test-support = [] diff --git a/rust/shared/src/crd/mod.rs b/rust/shared/src/crd/mod.rs new file mode 100644 index 000000000..8a7c2fb7c --- /dev/null +++ b/rust/shared/src/crd/mod.rs @@ -0,0 +1 @@ +pub mod topic; diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs new file mode 100644 index 000000000..d2816c979 --- /dev/null +++ b/rust/shared/src/crd/topic.rs @@ -0,0 +1,282 @@ +use std::{collections::BTreeMap, str::FromStr}; + +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + agent::{deletion::v1alpha1::DeletionPolicy, operation::v1alpha1::ResourceOperation}, + attributed_string_type, + k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition, + kube::{CustomResource, ResourceExt}, + schemars::{self, JsonSchema}, + v2::{macros::attributed_string_type::Error as TopicNameError, types::operator::ClusterName}, + versioned::versioned, +}; + +/// Set by the agent on a `KafkaTopic` that deletes its topic. Removed by the agent after the +/// deletion, or by the operator once the KafkaCluster is gone, so no agent can remove it anymore. +pub const KAFKA_TOPIC_FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; + +attributed_string_type! { + TopicName, + "The name of a Kafka topic, not `.` or `..`", + "shop.orders", + (min_length = 1), + (max_length = 249), + (regex = r"^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$") +} + +#[versioned( + version(name = "v1alpha1"), + crates( + kube_core = "stackable_operator::kube::core", + kube_client = "stackable_operator::kube::client", + k8s_openapi = "stackable_operator::k8s_openapi", + schemars = "stackable_operator::schemars", + versioned = "stackable_operator::versioned" + ) +)] +pub mod versioned { + /// A Kafka topic managed by the agent of the referenced Kafka cluster. + /// + /// The referenced `KafkaCluster` must have `clusterConfig.managementAccess` enabled or this + /// resource will not be reconciled. + #[versioned(crd( + doc = "A Kafka topic, created and updated by the agent of the referenced Kafka cluster.", + group = "kafka.stackable.tech", + plural = "kafkatopics", + status = "KafkaTopicStatus", + selectable = ".spec.clusterRef.name", + namespaced + ))] + #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "has(self.topicName) == has(oldSelf.topicName) && (!has(self.topicName) || self.topicName == oldSelf.topicName)", + "message": "topicName is immutable" + }]))] + pub struct KafkaTopicSpec { + /// The KafkaCluster managing this topic. Immutable. + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "self == oldSelf", + "message": "clusterRef is immutable" + }]))] + pub cluster_ref: v1alpha1::KafkaClusterRef, + + /// Name of the topic in Kafka. Defaults to `metadata.name`. Immutable. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "!self.startsWith('__')", + "message": "topics starting with __ are internal to Kafka" + }]))] + pub topic_name: Option, + + /// Number of partitions. Can only be increased. Defaults to the broker setting + /// `num.partitions` when the topic is created. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub partitions: Option, + + /// Number of replicas of each partition. Changes are not applied to existing topics. + /// Defaults to the broker setting `default.replication.factor` when the topic is created. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub replication_factor: Option, + + /// Topic configs like `retention.ms`. All other topic-level configs are reset to the broker + /// defaults, except the replication throttles set by partition reassignments. + #[serde(default)] + pub config_overrides: BTreeMap, + + // no doc - docs in DeletionPolicy enum. + #[serde(default)] + pub deletion_policy: DeletionPolicy, + + /// Operations on this topic, independent of the `clusterOperation` of the KafkaCluster. + #[serde(default)] + pub topic_operation: ResourceOperation, + } + + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaClusterRef { + /// Name of a KafkaCluster in the namespace of this KafkaTopic. + pub name: ClusterName, + } + + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaTopicStatus { + /// `Ready` and `Synced`. + #[serde(default)] + pub conditions: Vec, + + /// The topic id in Kafka, as shown by `kafka-topics.sh`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_id: Option, + + /// The name of the topic in Kafka. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_name: Option, + + /// The number of partitions in Kafka. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub partitions: Option, + + /// The replication factor, not counting replicas that a running reassignment removes. + /// Not set if the partitions have different replication factors. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replication_factor: Option, + + /// The configs set on the topic in Kafka, without broker and default values. + #[serde(default)] + pub config: BTreeMap, + } +} + +impl v1alpha1::KafkaTopic { + /// The name of the topic in Kafka: `spec.topicName`, otherwise `metadata.name`. + /// + /// Fails if `metadata.name` is longer than a topic name may be. + pub fn topic_name(&self) -> Result { + match &self.spec.topic_name { + Some(topic_name) => Ok(topic_name.clone()), + None => TopicName::from_str(&self.name_any()), + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use stackable_operator::{kube::api::ObjectMeta, versioned::test_utils::RoundtripTestData}; + + use super::*; + + impl RoundtripTestData for v1alpha1::KafkaTopicSpec { + fn roundtrip_test_data() -> Vec { + stackable_operator::utils::yaml_from_str_singleton_map(indoc::indoc! {r#" + - clusterRef: + name: simple-kafka + topicName: shop.orders + partitions: 6 + replicationFactor: 3 + configOverrides: + retention.ms: "604800000" + deletionPolicy: Delete + topicOperation: + reconciliationPaused: true + - clusterRef: + name: simple-kafka + "#}) + .expect("the roundtrip test data should be valid YAML") + } + } + + fn topic(metadata_name: &str, topic_name: Option<&str>) -> v1alpha1::KafkaTopic { + v1alpha1::KafkaTopic { + metadata: ObjectMeta { + name: Some(metadata_name.to_owned()), + ..ObjectMeta::default() + }, + spec: v1alpha1::KafkaTopicSpec { + cluster_ref: v1alpha1::KafkaClusterRef { + name: ClusterName::from_str("simple-kafka") + .expect("the cluster name should be valid"), + }, + topic_name: topic_name.map(TopicName::from_str_unsafe), + partitions: None, + replication_factor: None, + config_overrides: BTreeMap::new(), + deletion_policy: DeletionPolicy::default(), + topic_operation: ResourceOperation::default(), + }, + status: None, + } + } + + #[rstest] + #[case::dotted("shop.orders")] + #[case::uppercase_and_underscore("Shop_Orders-1")] + #[case::single_underscore("_internal")] + #[case::underscore_only("_")] + #[case::internal_topic("__consumer_offsets")] + #[case::leading_dot(".hidden")] + #[case::trailing_dot("a.")] + #[case::three_dots("...")] + fn accept_topic_name(#[case] name: &str) { + assert!( + TopicName::from_str(name).is_ok(), + "{name:?} should be valid" + ); + } + + #[rstest] + #[case::empty("")] + #[case::dot(".")] + #[case::two_dots("..")] + #[case::invalid_character("shop/orders")] + #[case::too_long(&"a".repeat(250))] + fn reject_topic_name(#[case] name: &str) { + assert!( + TopicName::from_str(name).is_err(), + "{name:?} should be invalid" + ); + } + + #[rstest] + #[case::explicit("my-topic", Some("shop.orders"), "shop.orders")] + #[case::from_metadata_name("shop.orders", None, "shop.orders")] + fn effective_topic_name( + #[case] metadata_name: &str, + #[case] topic_name: Option<&str>, + #[case] expected: &str, + ) { + assert_eq!( + TopicName::from_str_unsafe(expected), + topic(metadata_name, topic_name) + .topic_name() + .expect("the topic name should be valid") + ); + } + + #[test] + fn reject_too_long_metadata_name_as_topic_name() { + assert!(topic(&"a".repeat(250), None).topic_name().is_err()); + } + + #[test] + fn schema_contains_validations() { + let crd = serde_json::to_value( + KafkaTopic::merged_crd(KafkaTopicVersion::V1Alpha1) + .expect("the CRD should be mergeable"), + ) + .expect("the CRD should be serializable"); + let version = &crd["spec"]["versions"][0]; + let spec = &version["schema"]["openAPIV3Schema"]["properties"]["spec"]; + let properties = &spec["properties"]; + + assert_eq!( + ".spec.clusterRef.name", + version["selectableFields"][0]["jsonPath"] + ); + assert_eq!( + "self == oldSelf", + properties["clusterRef"]["x-kubernetes-validations"][0]["rule"] + ); + assert_eq!( + "has(self.topicName) == has(oldSelf.topicName) && (!has(self.topicName) || \ + self.topicName == oldSelf.topicName)", + spec["x-kubernetes-validations"][0]["rule"] + ); + assert_eq!(249, properties["topicName"]["maxLength"]); + assert_eq!( + "!self.startsWith('__')", + properties["topicName"]["x-kubernetes-validations"][0]["rule"] + ); + assert!(properties["topicName"]["pattern"].is_string()); + assert_eq!(Some(1.0), properties["partitions"]["minimum"].as_f64()); + assert_eq!( + Some(1.0), + properties["replicationFactor"]["minimum"].as_f64() + ); + } +} diff --git a/rust/shared/src/lib.rs b/rust/shared/src/lib.rs index f17bc55db..0e0beca04 100644 --- a/rust/shared/src/lib.rs +++ b/rust/shared/src/lib.rs @@ -1 +1,2 @@ pub mod agent; +pub mod crd; diff --git a/tests/templates/kuttl/management-access/00-patch-ns.yaml.j2 b/tests/templates/kuttl/management-access/00-patch-ns.yaml.j2 new file mode 100644 index 000000000..67185acfd --- /dev/null +++ b/tests/templates/kuttl/management-access/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/management-access/05-create-agent-credential.yaml.j2 b/tests/templates/kuttl/management-access/05-create-agent-credential.yaml.j2 new file mode 100644 index 000000000..fa4242f98 --- /dev/null +++ b/tests/templates/kuttl/management-access/05-create-agent-credential.yaml.j2 @@ -0,0 +1,29 @@ +{% if test_scenario['values']['management-access-authentication'] == 'secret' %} +--- +# A CA trusted by the brokers for client authentication, and a certificate for the agent signed by +# it, as a user would provide them +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + set -eu + dir=$(mktemp -d) + printf "basicConstraints=CA:FALSE\nextendedKeyUsage=clientAuth\n" > "$dir/extensions.cnf" + openssl req -x509 -newkey rsa:2048 -nodes -days 30 \ + -subj "/CN=Test client CA" \ + -addext "basicConstraints=critical,CA:TRUE" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -keyout "$dir/ca.key" -out "$dir/ca.crt" + openssl req -newkey rsa:2048 -nodes \ + -subj "/CN=kafka-agent" \ + -keyout "$dir/tls.key" -out "$dir/tls.csr" + openssl x509 -req -days 30 -in "$dir/tls.csr" \ + -CA "$dir/ca.crt" -CAkey "$dir/ca.key" -CAcreateserial \ + -extfile "$dir/extensions.cnf" \ + -out "$dir/tls.crt" + kubectl create secret generic client-auth-ca --namespace "$NAMESPACE" \ + --from-file=ca.crt="$dir/ca.crt" --from-file=ca.key="$dir/ca.key" + kubectl create secret tls kafka-agent-credential --namespace "$NAMESPACE" \ + --cert="$dir/tls.crt" --key="$dir/tls.key" + rm -r "$dir" +{% endif %} diff --git a/tests/templates/kuttl/management-access/10-assert.yaml b/tests/templates/kuttl/management-access/10-assert.yaml new file mode 100644 index 000000000..bc622bead --- /dev/null +++ b/tests/templates/kuttl/management-access/10-assert.yaml @@ -0,0 +1,21 @@ +--- +# The KafkaCluster and its agent are running. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-kafka-agent +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/management-access/10-install-kafka.yaml b/tests/templates/kuttl/management-access/10-install-kafka.yaml new file mode 100644 index 000000000..4652b959e --- /dev/null +++ b/tests/templates/kuttl/management-access/10-install-kafka.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: > + envsubst '$NAMESPACE' < 10_kafka.yaml | + kubectl apply --namespace $NAMESPACE -f - diff --git a/tests/templates/kuttl/management-access/10_kafka.yaml.j2 b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 new file mode 100644 index 000000000..ca2976982 --- /dev/null +++ b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 @@ -0,0 +1,78 @@ +{% set authentication = test_scenario['values']['management-access-authentication'] %} +{% if authentication != 'anonymous' %} +--- +# SecretClasses and AuthenticationClasses are cluster-scoped, so the namespace keeps their names +# unique between parallel test runs. +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: client-auth-$NAMESPACE +spec: + backend: + autoTls: + ca: + secret: + name: client-auth-ca + namespace: $NAMESPACE +{% if authentication == 'secret' %} + autoGenerate: false +{% else %} + autoGenerate: true +{% endif %} +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: client-auth-$NAMESPACE +spec: + provider: + tls: + clientCertSecretClass: client-auth-$NAMESPACE +{% endif %} +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + image: +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls +{% if authentication != 'anonymous' %} + authentication: + - authenticationClass: client-auth-$NAMESPACE +{% endif %} + managementAccess: + enabled: true + authentication: + userProvided: +{% if authentication == 'anonymous' %} + anonymous: {} +{% elif authentication == 'secret-class' %} + tls: + secretClassName: client-auth-$NAMESPACE +{% else %} + tls: + secretName: kafka-agent-credential +{% endif %} + controllers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 + brokers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/management-access/11-assert.yaml.j2 b/tests/templates/kuttl/management-access/11-assert.yaml.j2 new file mode 100644 index 000000000..1853fe47e --- /dev/null +++ b/tests/templates/kuttl/management-access/11-assert.yaml.j2 @@ -0,0 +1,16 @@ +{% set authentication = test_scenario['values']['management-access-authentication'] %} +--- +# Runs after the agent is ready, so the commands are not polled during the startup. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +commands: +{% if authentication == 'anonymous' %} + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "tls-kafka" + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_CLIENT_SECURITY")].value}')" = tls +{% else %} + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "tls-kafka tls-management-access" + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_CLIENT_SECURITY")].value}')" = mutual-tls +{% endif %} + # The headless Service keeps the traffic within the Kubernetes cluster. + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_BOOTSTRAP_SERVERS")].value}')" = "test-kafka-broker-default-headless.$NAMESPACE.svc.cluster.local:9093" diff --git a/tests/templates/kuttl/management-access/20-assert.yaml b/tests/templates/kuttl/management-access/20-assert.yaml new file mode 100644 index 000000000..c062e63ac --- /dev/null +++ b/tests/templates/kuttl/management-access/20-assert.yaml @@ -0,0 +1,22 @@ +--- +# The agent authenticates to Kafka and creates the topic. The KafkaTopic keeps its finalizer until +# the end of the test, so the deletion of the namespace only finishes if the operator removes it. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders + finalizers: + - kafka.stackable.tech/kafkatopic +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + partitions: 3 diff --git a/tests/templates/kuttl/management-access/20-create-topic.yaml b/tests/templates/kuttl/management-access/20-create-topic.yaml new file mode 100644 index 000000000..9da42728f --- /dev/null +++ b/tests/templates/kuttl/management-access/20-create-topic.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterRef: + name: test-kafka + partitions: 3 + deletionPolicy: Delete diff --git a/tests/templates/kuttl/management-access/kafka-cli.sh b/tests/templates/kuttl/management-access/kafka-cli.sh new file mode 100644 index 000000000..316274375 --- /dev/null +++ b/tests/templates/kuttl/management-access/kafka-cli.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Usage: kafka-cli.sh , e.g. kafka-cli.sh kafka-topics.sh --describe --topic orders +# Runs a Kafka CLI tool in the broker Pod against the brokers. +set -euo pipefail + +# The script is expanded in the broker Pod, not here. +# shellcheck disable=SC2016 +kubectl exec --namespace "$NAMESPACE" test-kafka-broker-default-0 --container kafka -- bash -c ' + bootstrap="$(cat /stackable/listener-broker/default-address/address):$(cat /stackable/listener-broker/default-address/ports/kafka-tls)" + tool=$1 + shift + "/stackable/kafka/bin/$tool" --bootstrap-server "$bootstrap" --command-config /stackable/config/client.properties "$@" +' kafka-cli "$@" diff --git a/tests/templates/kuttl/topic/00-patch-ns.yaml.j2 b/tests/templates/kuttl/topic/00-patch-ns.yaml.j2 new file mode 100644 index 000000000..67185acfd --- /dev/null +++ b/tests/templates/kuttl/topic/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/topic/10-assert.yaml b/tests/templates/kuttl/topic/10-assert.yaml new file mode 100644 index 000000000..bd9419b21 --- /dev/null +++ b/tests/templates/kuttl/topic/10-assert.yaml @@ -0,0 +1,43 @@ +--- +# The KafkaCluster and its agent are running. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-controller-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-kafka-agent +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-kafka-agent +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-kafka-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kafka-agent-clusterrole diff --git a/tests/templates/kuttl/topic/10-install-kafka.yaml.j2 b/tests/templates/kuttl/topic/10-install-kafka.yaml.j2 new file mode 100644 index 000000000..e563db1eb --- /dev/null +++ b/tests/templates/kuttl/topic/10-install-kafka.yaml.j2 @@ -0,0 +1,35 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + image: +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls + managementAccess: + enabled: true + authentication: + userProvided: + anonymous: {} + controllers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 + brokers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/topic/20-assert.yaml b/tests/templates/kuttl/topic/20-assert.yaml new file mode 100644 index 000000000..7290dfc03 --- /dev/null +++ b/tests/templates/kuttl/topic/20-assert.yaml @@ -0,0 +1,32 @@ +--- +# With the default deletionPolicy Retain, the KafkaTopic has no finalizer. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders + ref: orders +assertAll: + - celExpr: orders.status.topicId != '' + - celExpr: "!has(orders.metadata.finalizers)" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + topicName: orders + partitions: 3 + replicationFactor: 1 + config: + retention.ms: "3600000" + max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/20-create-topic.yaml b/tests/templates/kuttl/topic/20-create-topic.yaml new file mode 100644 index 000000000..2cba7e028 --- /dev/null +++ b/tests/templates/kuttl/topic/20-create-topic.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterRef: + name: test-kafka + # Equal to the name on purpose, so that step 31 can test that topicName is immutable. + topicName: orders + partitions: 3 + replicationFactor: 1 + configOverrides: + retention.ms: "3600000" + max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/21-assert.yaml b/tests/templates/kuttl/topic/21-assert.yaml new file mode 100644 index 000000000..fa8996d99 --- /dev/null +++ b/tests/templates/kuttl/topic/21-assert.yaml @@ -0,0 +1,21 @@ +--- +# The broker defaults num.partitions and default.replication.factor are both 1. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + topicName: defaults + partitions: 1 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/21-create-defaults-topic.yaml b/tests/templates/kuttl/topic/21-create-defaults-topic.yaml new file mode 100644 index 000000000..bfac2b989 --- /dev/null +++ b/tests/templates/kuttl/topic/21-create-defaults-topic.yaml @@ -0,0 +1,10 @@ +--- +# Without topicName, partitions and replicationFactor: the topic is named after the KafkaTopic and +# gets the broker defaults. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +spec: + clusterRef: + name: test-kafka diff --git a/tests/templates/kuttl/topic/22-reject-internal-topic.yaml b/tests/templates/kuttl/topic/22-reject-internal-topic.yaml new file mode 100644 index 000000000..a2518122b --- /dev/null +++ b/tests/templates/kuttl/topic/22-reject-internal-topic.yaml @@ -0,0 +1,8 @@ +--- +# The API server rejects topics starting with __, which are internal to Kafka. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + kubectl apply --namespace $NAMESPACE --filename 22_internal_topic.yaml 2>&1 + | grep "topics starting with __ are internal to Kafka" diff --git a/tests/templates/kuttl/topic/22_internal_topic.yaml b/tests/templates/kuttl/topic/22_internal_topic.yaml new file mode 100644 index 000000000..8003b8509 --- /dev/null +++ b/tests/templates/kuttl/topic/22_internal_topic.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: consumer-offsets +spec: + clusterRef: + name: test-kafka + topicName: __consumer_offsets diff --git a/tests/templates/kuttl/topic/30-assert.yaml b/tests/templates/kuttl/topic/30-assert.yaml new file mode 100644 index 000000000..7d407115b --- /dev/null +++ b/tests/templates/kuttl/topic/30-assert.yaml @@ -0,0 +1,21 @@ +--- +# The agent adds the partitions and applies the changed config. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + partitions: 5 + config: + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/30-change-topic.yaml b/tests/templates/kuttl/topic/30-change-topic.yaml new file mode 100644 index 000000000..911cc3843 --- /dev/null +++ b/tests/templates/kuttl/topic/30-change-topic.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + partitions: 5 + configOverrides: + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/31-reject-topic-name-change.yaml b/tests/templates/kuttl/topic/31-reject-topic-name-change.yaml new file mode 100644 index 000000000..14d5e3af7 --- /dev/null +++ b/tests/templates/kuttl/topic/31-reject-topic-name-change.yaml @@ -0,0 +1,22 @@ +--- +# The API server rejects changing, removing or adding topicName. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + kubectl patch kafkatopic orders --namespace $NAMESPACE --type merge + --patch '{"spec": {"topicName": "renamed"}}' 2>&1 + | grep "topicName is immutable" + - script: >- + kubectl patch kafkatopic orders --namespace $NAMESPACE --type json + --patch '[{"op": "remove", "path": "/spec/topicName"}]' 2>&1 + | grep "topicName is immutable" + - script: >- + kubectl patch kafkatopic defaults --namespace $NAMESPACE --type merge + --patch '{"spec": {"topicName": "defaults"}}' 2>&1 + | grep "topicName is immutable" + - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.spec.topicName}')" = orders + # An explicit null means the same as no topicName. + - script: >- + kubectl patch kafkatopic defaults --namespace $NAMESPACE --type json + --patch '[{"op": "add", "path": "/spec/topicName", "value": null}]' diff --git a/tests/templates/kuttl/topic/40-set-config-out-of-band.yaml b/tests/templates/kuttl/topic/40-set-config-out-of-band.yaml new file mode 100644 index 000000000..2a125a288 --- /dev/null +++ b/tests/templates/kuttl/topic/40-set-config-out-of-band.yaml @@ -0,0 +1,6 @@ +--- +# A config set by someone else than the agent, e.g. an administrator with kafka-configs.sh +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: bash kafka-cli.sh kafka-configs.sh --alter --entity-type topics --entity-name orders --add-config segment.ms=86400000 diff --git a/tests/templates/kuttl/topic/41-assert.yaml b/tests/templates/kuttl/topic/41-assert.yaml new file mode 100644 index 000000000..5544c145f --- /dev/null +++ b/tests/templates/kuttl/topic/41-assert.yaml @@ -0,0 +1,29 @@ +--- +# The configOverrides replace all topic-level configs: the key removed from the spec and the key +# set out of band are both reset to the default. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders + ref: orders +assertAll: + - celExpr: "!('max.message.bytes' in orders.status.config)" + - celExpr: "!('segment.ms' in orders.status.config)" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/41-remove-config-override.yaml b/tests/templates/kuttl/topic/41-remove-config-override.yaml new file mode 100644 index 000000000..31e7866f3 --- /dev/null +++ b/tests/templates/kuttl/topic/41-remove-config-override.yaml @@ -0,0 +1,9 @@ +--- +# kuttl merges applied objects into the existing ones, so a removed key has to be set to null. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + configOverrides: + max.message.bytes: null diff --git a/tests/templates/kuttl/topic/50-assert.yaml b/tests/templates/kuttl/topic/50-assert.yaml new file mode 100644 index 000000000..b7945043f --- /dev/null +++ b/tests/templates/kuttl/topic/50-assert.yaml @@ -0,0 +1,20 @@ +--- +# The topic stays usable and unchanged. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "False" + reason: UnsupportedChange + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/50-unsupported-change.yaml b/tests/templates/kuttl/topic/50-unsupported-change.yaml new file mode 100644 index 000000000..9831ad248 --- /dev/null +++ b/tests/templates/kuttl/topic/50-unsupported-change.yaml @@ -0,0 +1,9 @@ +--- +# Partitions cannot be removed, and the agent does not reassign replicas. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + partitions: 2 + replicationFactor: 3 diff --git a/tests/templates/kuttl/topic/51-assert.yaml b/tests/templates/kuttl/topic/51-assert.yaml new file mode 100644 index 000000000..2c43c1564 --- /dev/null +++ b/tests/templates/kuttl/topic/51-assert.yaml @@ -0,0 +1,20 @@ +--- +# The topic is in sync again once the spec matches it. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml b/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml new file mode 100644 index 000000000..cfaeab1f0 --- /dev/null +++ b/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/60-assert.yaml b/tests/templates/kuttl/topic/60-assert.yaml new file mode 100644 index 000000000..a2ae21b5c --- /dev/null +++ b/tests/templates/kuttl/topic/60-assert.yaml @@ -0,0 +1,22 @@ +--- +# Ready is unknown, because the agent did not look at the topic in Kafka. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: | + topics=$(bash kafka-cli.sh kafka-topics.sh --list) + ! echo "$topics" | grep -x throttled +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: throttled +status: + conditions: + - type: Ready + status: Unknown + reason: InvalidSpec + - type: Synced + status: "False" + reason: InvalidSpec diff --git a/tests/templates/kuttl/topic/60-invalid-spec.yaml b/tests/templates/kuttl/topic/60-invalid-spec.yaml new file mode 100644 index 000000000..85c919ced --- /dev/null +++ b/tests/templates/kuttl/topic/60-invalid-spec.yaml @@ -0,0 +1,11 @@ +--- +# Throttles are set by partition reassignments and cannot be managed by a KafkaTopic. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: throttled +spec: + clusterRef: + name: test-kafka + configOverrides: + leader.replication.throttled.replicas: "*" diff --git a/tests/templates/kuttl/topic/70-create-foreign-topic.yaml b/tests/templates/kuttl/topic/70-create-foreign-topic.yaml new file mode 100644 index 000000000..9591c9ca4 --- /dev/null +++ b/tests/templates/kuttl/topic/70-create-foreign-topic.yaml @@ -0,0 +1,8 @@ +--- +# A topic created outside of KafkaTopics, e.g. by an application or an administrator +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + bash kafka-cli.sh kafka-topics.sh --create --topic legacy --partitions 1 --replication-factor 1 + --config cleanup.policy=compact diff --git a/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml b/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml new file mode 100644 index 000000000..10395380c --- /dev/null +++ b/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: legacy +spec: + clusterRef: + name: test-kafka + deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/71-assert.yaml b/tests/templates/kuttl/topic/71-assert.yaml new file mode 100644 index 000000000..6da767ba8 --- /dev/null +++ b/tests/templates/kuttl/topic/71-assert.yaml @@ -0,0 +1,29 @@ +--- +# Existing topics are adopted, and their topic-level configs are replaced. With the deletionPolicy +# Delete, the adopted topic gets a finalizer. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: legacy + ref: legacy +assertAll: + - celExpr: legacy.status.topicId != '' + - celExpr: "!has(legacy.status.config) || !('cleanup.policy' in legacy.status.config)" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: legacy + finalizers: + - kafka.stackable.tech/kafkatopic +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced diff --git a/tests/templates/kuttl/topic/75-assert.yaml b/tests/templates/kuttl/topic/75-assert.yaml new file mode 100644 index 000000000..6a856dcf0 --- /dev/null +++ b/tests/templates/kuttl/topic/75-assert.yaml @@ -0,0 +1,20 @@ +--- +# The status still shows the topic in Kafka, with the old config. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "False" + reason: ReconciliationPaused + config: + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/75-pause-topic.yaml b/tests/templates/kuttl/topic/75-pause-topic.yaml new file mode 100644 index 000000000..f66a88641 --- /dev/null +++ b/tests/templates/kuttl/topic/75-pause-topic.yaml @@ -0,0 +1,11 @@ +--- +# The agent only observes a paused topic, so the changed config is not applied. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + configOverrides: + retention.ms: "10800000" + topicOperation: + reconciliationPaused: true diff --git a/tests/templates/kuttl/topic/76-assert.yaml b/tests/templates/kuttl/topic/76-assert.yaml new file mode 100644 index 000000000..b82d73e2d --- /dev/null +++ b/tests/templates/kuttl/topic/76-assert.yaml @@ -0,0 +1,20 @@ +--- +# After resuming, the agent applies the config changed while paused. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + retention.ms: "10800000" diff --git a/tests/templates/kuttl/topic/76-resume-topic.yaml b/tests/templates/kuttl/topic/76-resume-topic.yaml new file mode 100644 index 000000000..f20505613 --- /dev/null +++ b/tests/templates/kuttl/topic/76-resume-topic.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + topicOperation: + reconciliationPaused: false diff --git a/tests/templates/kuttl/topic/77-assert.yaml b/tests/templates/kuttl/topic/77-assert.yaml new file mode 100644 index 000000000..e60dcb2cd --- /dev/null +++ b/tests/templates/kuttl/topic/77-assert.yaml @@ -0,0 +1,25 @@ +--- +# The topic stays available, and the rejected value is not set on the topic. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: defaults + ref: defaults +assertAll: + - celExpr: "!has(defaults.status.config) || !('retention.ms' in defaults.status.config)" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "False" + reason: KafkaRejected diff --git a/tests/templates/kuttl/topic/77-invalid-config-value.yaml b/tests/templates/kuttl/topic/77-invalid-config-value.yaml new file mode 100644 index 000000000..f23d4445f --- /dev/null +++ b/tests/templates/kuttl/topic/77-invalid-config-value.yaml @@ -0,0 +1,9 @@ +--- +# Kafka rejects the config value, which only it can validate. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +spec: + configOverrides: + retention.ms: "abc" diff --git a/tests/templates/kuttl/topic/80-assert.yaml b/tests/templates/kuttl/topic/80-assert.yaml new file mode 100644 index 000000000..c22dd5c05 --- /dev/null +++ b/tests/templates/kuttl/topic/80-assert.yaml @@ -0,0 +1,41 @@ +--- +# The older KafkaTopic keeps managing the topic. The newer one gets no finalizer, so deleting it +# does not delete the topic. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders-copy + ref: orders_copy +assertAll: + - celExpr: "!has(orders_copy.metadata.finalizers)" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders-copy +status: + conditions: + - type: Ready + status: "False" + reason: Conflict + - type: Synced + status: "False" + reason: Conflict +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + retention.ms: "10800000" diff --git a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml new file mode 100644 index 000000000..b213bb82c --- /dev/null +++ b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml @@ -0,0 +1,11 @@ +--- +# A second KafkaTopic for the topic already managed by the KafkaTopic orders +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders-copy +spec: + clusterRef: + name: test-kafka + topicName: orders + deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/81-assert.yaml b/tests/templates/kuttl/topic/81-assert.yaml new file mode 100644 index 000000000..1e7d88dc6 --- /dev/null +++ b/tests/templates/kuttl/topic/81-assert.yaml @@ -0,0 +1,7 @@ +--- +# Deleting the conflicting KafkaTopic leaves the topic in Kafka. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash kafka-cli.sh kafka-topics.sh --list | grep -x orders diff --git a/tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml b/tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml new file mode 100644 index 000000000..a9805a169 --- /dev/null +++ b/tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders-copy diff --git a/tests/templates/kuttl/topic/91-assert.yaml b/tests/templates/kuttl/topic/91-assert.yaml new file mode 100644 index 000000000..537525c93 --- /dev/null +++ b/tests/templates/kuttl/topic/91-assert.yaml @@ -0,0 +1,11 @@ +--- +# orders has the default deletion policy Retain, legacy has Delete. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: test -z "$(kubectl get kafkatopics --namespace $NAMESPACE --output name)" + - script: bash kafka-cli.sh kafka-topics.sh --list | grep -x orders + - script: | + topics=$(bash kafka-cli.sh kafka-topics.sh --list) + ! echo "$topics" | grep -x legacy diff --git a/tests/templates/kuttl/topic/91-delete-topics.yaml b/tests/templates/kuttl/topic/91-delete-topics.yaml new file mode 100644 index 000000000..5de81d40e --- /dev/null +++ b/tests/templates/kuttl/topic/91-delete-topics.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: legacy + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: throttled + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: defaults diff --git a/tests/templates/kuttl/topic/92-assert.yaml b/tests/templates/kuttl/topic/92-assert.yaml new file mode 100644 index 000000000..64caeabbd --- /dev/null +++ b/tests/templates/kuttl/topic/92-assert.yaml @@ -0,0 +1,20 @@ +--- +# While the agent is still running, it creates the topic and adds the finalizer. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup + finalizers: + - kafka.stackable.tech/kafkatopic +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced diff --git a/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml b/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml new file mode 100644 index 000000000..6be86881a --- /dev/null +++ b/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml @@ -0,0 +1,11 @@ +--- +# From here on, the operator takes over the KafkaTopic, because the agent is removed in the +# following steps. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup +spec: + clusterRef: + name: test-kafka + deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/95-assert.yaml b/tests/templates/kuttl/topic/95-assert.yaml new file mode 100644 index 000000000..9df1148ed --- /dev/null +++ b/tests/templates/kuttl/topic/95-assert.yaml @@ -0,0 +1,18 @@ +--- +# Without an agent, the operator sets the conditions of the remaining KafkaTopics. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup +status: + conditions: + - type: Ready + status: Unknown + reason: AgentUnavailable + - type: Synced + status: "False" + reason: AgentUnavailable diff --git a/tests/templates/kuttl/topic/95-disable-management-access.yaml b/tests/templates/kuttl/topic/95-disable-management-access.yaml new file mode 100644 index 000000000..a48f7063a --- /dev/null +++ b/tests/templates/kuttl/topic/95-disable-management-access.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + clusterConfig: + managementAccess: + enabled: false diff --git a/tests/templates/kuttl/topic/95-errors.yaml b/tests/templates/kuttl/topic/95-errors.yaml new file mode 100644 index 000000000..da7eefef5 --- /dev/null +++ b/tests/templates/kuttl/topic/95-errors.yaml @@ -0,0 +1,16 @@ +--- +# The agent resources are removed when management access is disabled. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-kafka-agent +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-kafka-agent +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-kafka-agent diff --git a/tests/templates/kuttl/topic/96-assert.yaml b/tests/templates/kuttl/topic/96-assert.yaml new file mode 100644 index 000000000..d99243f83 --- /dev/null +++ b/tests/templates/kuttl/topic/96-assert.yaml @@ -0,0 +1,15 @@ +--- +# The operator keeps the finalizer while the KafkaCluster exists, so the deletion waits for an +# agent. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: sleep 30 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: cleanup + ref: cleanup +assertAll: + - celExpr: has(cleanup.metadata.deletionTimestamp) diff --git a/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml b/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml new file mode 100644 index 000000000..4962acaaf --- /dev/null +++ b/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml @@ -0,0 +1,6 @@ +--- +# Not a kuttl delete, as that waits for the deletion, which only finishes with the KafkaCluster. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl delete kafkatopic cleanup --namespace $NAMESPACE --wait=false diff --git a/tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml b/tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml new file mode 100644 index 000000000..039bc529d --- /dev/null +++ b/tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + name: test-kafka diff --git a/tests/templates/kuttl/topic/97-errors.yaml b/tests/templates/kuttl/topic/97-errors.yaml new file mode 100644 index 000000000..4e62a73a6 --- /dev/null +++ b/tests/templates/kuttl/topic/97-errors.yaml @@ -0,0 +1,6 @@ +--- +# Without the KafkaCluster, the operator removes the finalizer, so the deletion finishes. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup diff --git a/tests/templates/kuttl/topic/98-keep-environment.yaml b/tests/templates/kuttl/topic/98-keep-environment.yaml new file mode 100644 index 000000000..910bec3e8 --- /dev/null +++ b/tests/templates/kuttl/topic/98-keep-environment.yaml @@ -0,0 +1,7 @@ +--- +# With --skip-delete, the namespace is kept, so the deleted cluster is created again to experiment +# with. See the script for the details. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: bash keep-environment.sh diff --git a/tests/templates/kuttl/topic/98_topic.yaml b/tests/templates/kuttl/topic/98_topic.yaml new file mode 100644 index 000000000..d8aa89eaf --- /dev/null +++ b/tests/templates/kuttl/topic/98_topic.yaml @@ -0,0 +1,9 @@ +--- +# Adopts the topic that the KafkaTopic orders retained in Kafka. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterRef: + name: test-kafka diff --git a/tests/templates/kuttl/topic/kafka-cli.sh b/tests/templates/kuttl/topic/kafka-cli.sh new file mode 100644 index 000000000..316274375 --- /dev/null +++ b/tests/templates/kuttl/topic/kafka-cli.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Usage: kafka-cli.sh , e.g. kafka-cli.sh kafka-topics.sh --describe --topic orders +# Runs a Kafka CLI tool in the broker Pod against the brokers. +set -euo pipefail + +# The script is expanded in the broker Pod, not here. +# shellcheck disable=SC2016 +kubectl exec --namespace "$NAMESPACE" test-kafka-broker-default-0 --container kafka -- bash -c ' + bootstrap="$(cat /stackable/listener-broker/default-address/address):$(cat /stackable/listener-broker/default-address/ports/kafka-tls)" + tool=$1 + shift + "/stackable/kafka/bin/$tool" --bootstrap-server "$bootstrap" --command-config /stackable/config/client.properties "$@" +' kafka-cli "$@" diff --git a/tests/templates/kuttl/topic/keep-environment.sh b/tests/templates/kuttl/topic/keep-environment.sh new file mode 100644 index 000000000..fcd274f9e --- /dev/null +++ b/tests/templates/kuttl/topic/keep-environment.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Creates the KafkaCluster and a KafkaTopic again after the deletion steps, if kuttl was started +# with --skip-delete, so that the kept namespace has a running cluster to experiment with. kuttl +# does not expose the flag to test steps, hence looking for it in the argv of the kuttl ancestor. +set -euo pipefail + +pid=$PPID +while [ "${pid:-0}" -gt 1 ]; do + args=$(ps -o args= -p "$pid" 2>/dev/null || true) + case "$args" in + *kuttl*--skip-delete*) break ;; + esac + pid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d '[:space:]' || true) +done +if [ "${pid:-0}" -le 1 ]; then + echo "kuttl deletes the namespace, nothing to keep" + exit 0 +fi + +# The cluster id is the name of the KafkaCluster, so the topics in the kept PersistentVolumeClaims +# are back once the cluster runs. +kubectl apply --namespace "$NAMESPACE" --filename 10-install-kafka.yaml +kubectl apply --namespace "$NAMESPACE" --filename 98_topic.yaml diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index c4c294ffd..c3873d452 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -72,6 +72,12 @@ dimensions: - "cluster-internal" - "external-stable" - "external-unstable" + # How the agent authenticates to Kafka + - name: management-access-authentication + values: + - anonymous + - secret-class + - secret tests: - name: operations-kraft dimensions: @@ -147,6 +153,15 @@ tests: - openshift - broker-listener-class - bootstrap-listener-class + - name: topic + dimensions: + - kafka-kraft + - openshift + - name: management-access + dimensions: + - kafka-kraft + - management-access-authentication + - openshift suites: - name: nightly