diff --git a/CHANGELOG.md b/CHANGELOG.md index 91293ac0..398061e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file. `restarter.stackable.tech/enabled: "true"` label, opting them into the restarter-controller so that their Pods are automatically rolled when a mounted ConfigMap or Secret changes ([#754]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#779]). +- Add support for Spark 4.x and HBase connectors ([#797]). ### Changed @@ -110,6 +111,7 @@ All notable changes to this project will be documented in this file. [#790]: https://github.com/stackabletech/spark-k8s-operator/pull/790 [#795]: https://github.com/stackabletech/spark-k8s-operator/pull/795 [#796]: https://github.com/stackabletech/spark-k8s-operator/pull/796 +[#797]: https://github.com/stackabletech/spark-k8s-operator/pull/797 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index f5ab5e66..1869accf 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -7,6 +7,6 @@ - 4.1.3 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (LTS) - 4.1.2 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (Deprecated) -Apache Spark 4.x has the following known issues (as of July 2026): - -- Missing HBase compatibility (See: https://issues.apache.org/jira/browse/HBASE-30178) +All Spark images also include the HBase connectors library built against HBase 2.6.7. +As of today (26th October 2026) there is no official release of these connectors with Spark 4 support. +The version included in the SDP Spark images is built from https://github.com/stackabletech/hbase-connectors/commit/07f5791c4921e471d328c8ce5f760ac72f0090cb[07f5791] on the `master` branch. diff --git a/tests/templates/kuttl/hbase-connector/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/hbase-connector/10-deploy-spark-app.yaml.j2 index 87aa996d..0fbfc9b9 100644 --- a/tests/templates/kuttl/hbase-connector/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/hbase-connector/10-deploy-spark-app.yaml.j2 @@ -88,7 +88,6 @@ data: spark._jvm.org.apache.hadoop.hbase.spark.HBaseContext( spark._jsc.sc(), spark._jvm.org.apache.hadoop.hbase.HBaseConfiguration.create(), - None, ) catalog = '{\ @@ -101,7 +100,8 @@ data: df\ .write\ - .format("org.apache.hadoop.hbase.spark")\ + .format("org.apache.hadoop.hbase.spark.datasources.HBaseTableProvider")\ + .mode("append")\ .option('catalog', catalog)\ .option('newtable', '5')\ .save() @@ -117,7 +117,7 @@ data: # scan, which is all this test needs. read_df = spark\ .read\ - .format("org.apache.hadoop.hbase.spark")\ + .format("org.apache.hadoop.hbase.spark.datasources.HBaseTableProvider")\ .option('catalog', catalog)\ .option('hbase.spark.pushdown.columnfilter', 'false')\ .load() diff --git a/tests/templates/kuttl/product-config-compat/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/product-config-compat/10-deploy-spark-app.yaml.j2 index 7d2dc88d..b4b535b8 100644 --- a/tests/templates/kuttl/product-config-compat/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/product-config-compat/10-deploy-spark-app.yaml.j2 @@ -111,7 +111,7 @@ spec: spark-defaults.conf: spark.driver.cores: "3" # Overrides the operator-set value; asserts that a server-level configOverride survives the merge with the executor properties. - spark.kubernetes.container.image.pullPolicy: "Always" + spark.ui.prometheus.enabled: "false" executor: configOverrides: spark-defaults.conf: diff --git a/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json b/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json index dc889b4e..21556c45 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json @@ -1,6 +1,6 @@ { "metrics.properties": "*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet\n*.sink.prometheusServlet.path=/metrics/prometheus\n", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", - "spark-defaults.conf": "spark.driver.cores=3\nspark.driver.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\\ -Dmy.custom.jvm.arg\\=customValue\nspark.driver.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-4.2.0.jar\nspark.driver.host=spark-connect-server-headless\nspark.executor.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\nspark.executor.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-4.2.0.jar\nspark.executor.instances=3\nspark.executor.memory=1024M\nspark.executor.memoryOverhead=1m\nspark.kubernetes.authenticate.driver.serviceAccountName=spark-connect-serviceaccount\nspark.kubernetes.container.image.pullPolicy=Always\nspark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s\\:4.2.0-stackable0.0.0-dev\nspark.kubernetes.driver.pod.name=${env\\:HOSTNAME}\nspark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s\\:4.2.0-stackable0.0.0-dev\nspark.kubernetes.executor.limit.cores=1\nspark.kubernetes.executor.podTemplateContainerName=spark\nspark.kubernetes.executor.podTemplateFile=/stackable/spark/conf/template.yaml\nspark.kubernetes.executor.request.cores=1\nspark.kubernetes.namespace=__NAMESPACE__\nspark.metrics.conf=/stackable/spark/conf/metrics.properties\nspark.ui.prometheus.enabled=true\n", + "spark-defaults.conf": "spark.driver.cores=3\nspark.driver.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\\ -Dmy.custom.jvm.arg\\=customValue\nspark.driver.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-4.2.0.jar\nspark.driver.host=spark-connect-server-headless\nspark.executor.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\nspark.executor.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-4.2.0.jar\nspark.executor.instances=3\nspark.executor.memory=1024M\nspark.executor.memoryOverhead=1m\nspark.kubernetes.authenticate.driver.serviceAccountName=spark-connect-serviceaccount\nspark.kubernetes.container.image.pullPolicy=IfNotPresent\nspark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s\\:4.2.0-stackable0.0.0-dev\nspark.kubernetes.driver.pod.name=${env\\:HOSTNAME}\nspark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s\\:4.2.0-stackable0.0.0-dev\nspark.kubernetes.executor.limit.cores=1\nspark.kubernetes.executor.podTemplateContainerName=spark\nspark.kubernetes.executor.podTemplateFile=/stackable/spark/conf/template.yaml\nspark.kubernetes.executor.request.cores=1\nspark.kubernetes.namespace=__NAMESPACE__\nspark.metrics.conf=/stackable/spark/conf/metrics.properties\nspark.ui.prometheus.enabled=false\n", "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/managed-by: spark.stackable.tech_connect\n app.kubernetes.io/name: spark-connect\n app.kubernetes.io/version: 4.2.0-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\nspec:\n affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - podAffinityTerm:\n labelSelector:\n matchLabels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/name: spark-connect\n topologyKey: kubernetes.io/hostname\n weight: 70\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n name: spark\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log\n name: log\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/truststore\n name: stackable-truststore\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n seccompProfile:\n type: RuntimeDefault\n volumes:\n - emptyDir:\n sizeLimit: 30Mi\n name: log\n - configMap:\n name: spark-connect-executor\n name: config\n - configMap:\n name: spark-connect-log-config\n name: log-config\n - emptyDir: {}\n name: stackable-truststore\n" } diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index bfd18f7a..bd23c4f6 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -16,11 +16,11 @@ dimensions: - 4.1.2 - 4.1.3 - 4.2.0 - # No hbase-connector release with support for Spark 4 yet, so the hbase-connector test is disabled - # until one is available. - # - name: spark-hbase-connector - # values: - # - 3.5.8 + - name: spark-hbase-connector + values: + # - 4.1.2 # skip this version to reduce the number of tests + - 4.1.3 + - 4.2.0 - name: spark-delta-lake values: - 4.1.2 @@ -56,7 +56,7 @@ dimensions: - "CLUSTER.LOCAL" - name: hbase values: - - 2.6.6 + - 2.6.7 - name: hdfs-latest values: - 3.5.0 @@ -136,13 +136,13 @@ tests: dimensions: - spark-delta-lake - openshift - # - name: hbase-connector - # dimensions: - # - spark-hbase-connector - # - hbase - # - hdfs-latest - # - zookeeper-latest - # - openshift + - name: hbase-connector + dimensions: + - spark-hbase-connector + - hbase + - hdfs-latest + - zookeeper-latest + - openshift - name: custom-log-directory dimensions: - spark