feat: Add HBase Connector support for Spark 4.x

CHANGELOG.md

@@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file.
   `restarter.stackable.tech/enabled: "true"` label, opting them into the restarter-controller so
   that their Pods are automatically rolled when a mounted ConfigMap or Secret changes ([#754]).
 - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#779]).
+- Add support for Spark 4.x and HBase connectors ([#797]).
 
 ### Changed
 
@@ -110,6 +111,7 @@ All notable changes to this project will be documented in this file.
 [#790]: https://github.com/stackabletech/spark-k8s-operator/pull/790
 [#795]: https://github.com/stackabletech/spark-k8s-operator/pull/795
 [#796]: https://github.com/stackabletech/spark-k8s-operator/pull/796
+[#797]: https://github.com/stackabletech/spark-k8s-operator/pull/797
 
 ## [26.7.0] - 2026-07-21
 

docs/modules/spark-k8s/partials/supported-versions.adoc

@@ -7,6 +7,6 @@
 - 4.1.3 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (LTS)
 - 4.1.2 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (Deprecated)
 
-Apache Spark 4.x has the following known issues (as of July 2026):
-
-- Missing HBase compatibility (See: https://issues.apache.org/jira/browse/HBASE-30178)
+All Spark images also include the HBase connectors library built against HBase 2.6.7.
+As of today (26th October 2026) there is no official release of these connectors with Spark 4 support.
+The version included in the SDP Spark images is built from https://github.com/stackabletech/hbase-connectors/commit/07f5791c4921e471d328c8ce5f760ac72f0090cb[07f5791] on the `master` branch.

tests/templates/kuttl/hbase-connector/10-deploy-spark-app.yaml.j2

@@ -88,7 +88,6 @@ data:
     spark._jvm.org.apache.hadoop.hbase.spark.HBaseContext(
       spark._jsc.sc(),
       spark._jvm.org.apache.hadoop.hbase.HBaseConfiguration.create(),
-      None,
     )
 
     catalog = '{\
@@ -101,7 +100,8 @@ data:
 
     df\
       .write\
-      .format("org.apache.hadoop.hbase.spark")\
+      .format("org.apache.hadoop.hbase.spark.datasources.HBaseTableProvider")\
+      .mode("append")\
       .option('catalog', catalog)\
       .option('newtable', '5')\
       .save()
@@ -117,7 +117,7 @@ data:
     # scan, which is all this test needs.
     read_df = spark\
       .read\
-      .format("org.apache.hadoop.hbase.spark")\
+      .format("org.apache.hadoop.hbase.spark.datasources.HBaseTableProvider")\
       .option('catalog', catalog)\
       .option('hbase.spark.pushdown.columnfilter', 'false')\
       .load()

tests/templates/kuttl/product-config-compat/10-deploy-spark-app.yaml.j2

@@ -111,7 +111,7 @@ spec:
       spark-defaults.conf:
         spark.driver.cores: "3"
         # Overrides the operator-set value; asserts that a server-level configOverride survives the merge with the executor properties.
-        spark.kubernetes.container.image.pullPolicy: "Always"
+        spark.ui.prometheus.enabled: "false"
   executor:
     configOverrides:
       spark-defaults.conf:

tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json

@@ -1,6 +1,6 @@
 {
   "metrics.properties": "*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet\n*.sink.prometheusServlet.path=/metrics/prometheus\n",
   "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n",
-  "spark-defaults.conf": "spark.driver.cores=3\nspark.driver.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\\ -Dmy.custom.jvm.arg\\=customValue\nspark.driver.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-4.2.0.jar\nspark.driver.host=spark-connect-server-headless\nspark.executor.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\nspark.executor.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-4.2.0.jar\nspark.executor.instances=3\nspark.executor.memory=1024M\nspark.executor.memoryOverhead=1m\nspark.kubernetes.authenticate.driver.serviceAccountName=spark-connect-serviceaccount\nspark.kubernetes.container.image.pullPolicy=Always\nspark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s\\:4.2.0-stackable0.0.0-dev\nspark.kubernetes.driver.pod.name=${env\\:HOSTNAME}\nspark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s\\:4.2.0-stackable0.0.0-dev\nspark.kubernetes.executor.limit.cores=1\nspark.kubernetes.executor.podTemplateContainerName=spark\nspark.kubernetes.executor.podTemplateFile=/stackable/spark/conf/template.yaml\nspark.kubernetes.executor.request.cores=1\nspark.kubernetes.namespace=__NAMESPACE__\nspark.metrics.conf=/stackable/spark/conf/metrics.properties\nspark.ui.prometheus.enabled=true\n",
+  "spark-defaults.conf": "spark.driver.cores=3\nspark.driver.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\\ -Dmy.custom.jvm.arg\\=customValue\nspark.driver.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-4.2.0.jar\nspark.driver.host=spark-connect-server-headless\nspark.executor.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\nspark.executor.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-4.2.0.jar\nspark.executor.instances=3\nspark.executor.memory=1024M\nspark.executor.memoryOverhead=1m\nspark.kubernetes.authenticate.driver.serviceAccountName=spark-connect-serviceaccount\nspark.kubernetes.container.image.pullPolicy=IfNotPresent\nspark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s\\:4.2.0-stackable0.0.0-dev\nspark.kubernetes.driver.pod.name=${env\\:HOSTNAME}\nspark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s\\:4.2.0-stackable0.0.0-dev\nspark.kubernetes.executor.limit.cores=1\nspark.kubernetes.executor.podTemplateContainerName=spark\nspark.kubernetes.executor.podTemplateFile=/stackable/spark/conf/template.yaml\nspark.kubernetes.executor.request.cores=1\nspark.kubernetes.namespace=__NAMESPACE__\nspark.metrics.conf=/stackable/spark/conf/metrics.properties\nspark.ui.prometheus.enabled=false\n",
   "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: executor\n    app.kubernetes.io/instance: spark-connect\n    app.kubernetes.io/managed-by: spark.stackable.tech_connect\n    app.kubernetes.io/name: spark-connect\n    app.kubernetes.io/version: 4.2.0-stackable0.0.0-dev\n    stackable.tech/vendor: Stackable\nspec:\n  affinity:\n    podAntiAffinity:\n      preferredDuringSchedulingIgnoredDuringExecution:\n      - podAffinityTerm:\n          labelSelector:\n            matchLabels:\n              app.kubernetes.io/component: executor\n              app.kubernetes.io/instance: spark-connect\n              app.kubernetes.io/name: spark-connect\n          topologyKey: kubernetes.io/hostname\n        weight: 70\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    name: spark\n    volumeMounts:\n    - mountPath: /stackable/spark/conf\n      name: config\n    - mountPath: /stackable/log\n      name: log\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/truststore\n      name: stackable-truststore\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n    runAsNonRoot: true\n    seccompProfile:\n      type: RuntimeDefault\n  volumes:\n  - emptyDir:\n      sizeLimit: 30Mi\n    name: log\n  - configMap:\n      name: spark-connect-executor\n    name: config\n  - configMap:\n      name: spark-connect-log-config\n    name: log-config\n  - emptyDir: {}\n    name: stackable-truststore\n"
 }

tests/test-definition.yaml

@@ -16,11 +16,11 @@ dimensions:
       - 4.1.2
       - 4.1.3
       - 4.2.0
-  # No hbase-connector release with support for Spark 4 yet, so the hbase-connector test is disabled
-  # until one is available.
-  # - name: spark-hbase-connector
-  #   values:
-  #     - 3.5.8
+  - name: spark-hbase-connector
+    values:
+      # - 4.1.2  # skip this version to reduce the number of tests
+      - 4.1.3
+      - 4.2.0
   - name: spark-delta-lake
     values:
       - 4.1.2
@@ -56,7 +56,7 @@ dimensions:
       - "CLUSTER.LOCAL"
   - name: hbase
     values:
-      - 2.6.6
+      - 2.6.7
   - name: hdfs-latest
     values:
       - 3.5.0
@@ -136,13 +136,13 @@ tests:
     dimensions:
       - spark-delta-lake
       - openshift
-  # - name: hbase-connector
-  #   dimensions:
-  #     - spark-hbase-connector
-  #     - hbase
-  #     - hdfs-latest
-  #     - zookeeper-latest
-  #     - openshift
+  - name: hbase-connector
+    dimensions:
+      - spark-hbase-connector
+      - hbase
+      - hdfs-latest
+      - zookeeper-latest
+      - openshift
   - name: custom-log-directory
     dimensions:
       - spark