From e118b00843d46ff3ef008e779e712a8c3d34e1c4 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Thu, 10 Sep 2026 18:50:57 +0000 Subject: [PATCH 1/2] Infra: Replace MinIO with RustFS in the dev and integration stacks MinIO Community Edition stopped publishing images in October 2025 and the repository was archived in April 2026, so `minio/minio:latest` resolves to a frozen build that will not receive further security fixes (#2664). I'm choosing RustFS since that's what Polaris uses. --- Makefile | 2 +- dev/docker-compose-integration.yml | 54 ++++++++++++++++++------------ dev/docker-compose.yml | 44 +++++++++++++----------- dev/hive/core-site.xml | 2 +- dev/{run-minio.sh => run-s3.sh} | 11 ++---- dev/spark/spark-defaults.conf | 6 ++-- mkdocs/docs/contributing.md | 4 +-- 7 files changed, 67 insertions(+), 56 deletions(-) rename dev/{run-minio.sh => run-s3.sh} (72%) diff --git a/Makefile b/Makefile index 9a73b240a7..7bbc05e9db 100644 --- a/Makefile +++ b/Makefile @@ -129,7 +129,7 @@ test-integration-rebuild: ## Rebuild integration Docker services from scratch docker compose -f dev/docker-compose-integration.yml build --no-cache test-s3: ## Run tests marked with @pytest.mark.s3 - sh ./dev/run-minio.sh + sh ./dev/run-s3.sh $(TEST_RUNNER) pytest tests/ -m s3 $(PYTEST_ARGS) test-adls: ## Run tests marked with @pytest.mark.adls diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 32cdccbb1a..b74daacc31 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -28,7 +28,7 @@ services: depends_on: - rest - hive - - minio + - rustfs ports: - 15002:15002 # Spark Connect - 4040:4040 # Spark UI @@ -39,7 +39,7 @@ services: links: - rest:rest - hive:hive - - minio:minio + - rustfs:rustfs healthcheck: test: ["CMD", "sh", "-c", "netstat -an | grep 15002 | grep LISTEN"] interval: 30s @@ -59,41 +59,51 @@ services: - AWS_REGION=us-east-1 - CATALOG_WAREHOUSE=s3://warehouse/ - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - - CATALOG_S3_ENDPOINT=http://minio:9000 + - CATALOG_S3_ENDPOINT=http://rustfs:9000 - CATALOG_JDBC_STRICT__MODE=true - minio: - image: minio/minio - container_name: pyiceberg-minio + rustfs: + image: rustfs/rustfs:1.0.0-rc.5 + container_name: pyiceberg-rustfs networks: iceberg_net: aliases: - - warehouse.minio + - warehouse.rustfs ports: - 9001:9001 - 9000:9000 environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=minio - command: ["server", "/data", "--console-address", ":9001"] - mc: - image: minio/mc - container_name: pyiceberg-mc + - RUSTFS_ACCESS_KEY=admin + - RUSTFS_SECRET_KEY=password + - RUSTFS_VOLUMES=/data + - RUSTFS_ADDRESS=:9000 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_CONSOLE_ADDRESS=:9001 + # Resolves virtual-hosted-style `warehouse.rustfs` to the warehouse bucket. + # The port is required until rustfs/rustfs#7051 ships. + - RUSTFS_SERVER_DOMAINS=rustfs:9000 + healthcheck: + test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 5s + aws-cli: + image: amazon/aws-cli:2.36.40 + container_name: pyiceberg-aws-cli networks: iceberg_net: depends_on: - - minio + rustfs: + condition: service_healthy environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password - AWS_REGION=us-east-1 - entrypoint: > - /bin/sh -c " - until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; - /usr/bin/mc mb minio/warehouse; - /usr/bin/mc policy set public minio/warehouse; - tail -f /dev/null - " + - AWS_ENDPOINT_URL=http://rustfs:9000 + entrypoint: /bin/sh + command: + - -c + - "aws s3 mb s3://warehouse" hive: image: pyiceberg-hive:latest build: diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 609ac8d51f..2db3572ad1 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -16,31 +16,37 @@ # under the License. services: - minio: - image: minio/minio - container_name: pyiceberg-minio + rustfs: + image: rustfs/rustfs:1.0.0-rc.5 + container_name: pyiceberg-rustfs environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=minio + - RUSTFS_ACCESS_KEY=admin + - RUSTFS_SECRET_KEY=password + - RUSTFS_VOLUMES=/data + - RUSTFS_ADDRESS=:9000 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_CONSOLE_ADDRESS=:9001 ports: - 9001:9001 - 9000:9000 - command: ["server", "/data", "--console-address", ":9001"] - mc: + healthcheck: + test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 5s + aws-cli: depends_on: - - minio - image: minio/mc - container_name: pyiceberg-mc + rustfs: + condition: service_healthy + image: amazon/aws-cli:2.36.40 + container_name: pyiceberg-aws-cli environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password - AWS_REGION=us-east-1 - entrypoint: > - /bin/sh -c " - until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; - /usr/bin/mc rm -r --force minio/warehouse; - /usr/bin/mc mb minio/warehouse; - /usr/bin/mc policy set public minio/warehouse; - exit 0; - " + - AWS_ENDPOINT_URL=http://rustfs:9000 + entrypoint: /bin/sh + command: + - -c + - "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse" diff --git a/dev/hive/core-site.xml b/dev/hive/core-site.xml index f5a9473b51..fea864f9d9 100644 --- a/dev/hive/core-site.xml +++ b/dev/hive/core-site.xml @@ -32,7 +32,7 @@ fs.s3a.endpoint - http://minio:9000 + http://rustfs:9000 fs.s3a.access.key diff --git a/dev/run-minio.sh b/dev/run-s3.sh similarity index 72% rename from dev/run-minio.sh rename to dev/run-s3.sh index 3e4a55c19a..7a37742890 100755 --- a/dev/run-minio.sh +++ b/dev/run-s3.sh @@ -20,14 +20,9 @@ set -ex -if [ $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]; then - echo "Minio backend running" +if [ $(docker ps -q --filter "name=pyiceberg-rustfs" --filter "health=healthy" ) ]; then + echo "S3 backend running" else docker compose -f dev/docker-compose.yml kill - docker compose -f dev/docker-compose.yml up -d - while [ -z $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ] - do - echo "Waiting for Minio" - sleep 1 - done + docker compose -f dev/docker-compose.yml up -d --wait fi diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf index 847e733708..fcf40390ba 100644 --- a/dev/spark/spark-defaults.conf +++ b/dev/spark/spark-defaults.conf @@ -23,7 +23,7 @@ spark.sql.catalog.rest.type rest spark.sql.catalog.rest.uri http://rest:8181 spark.sql.catalog.rest.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.rest.warehouse s3://warehouse/rest/ -spark.sql.catalog.rest.s3.endpoint http://minio:9000 +spark.sql.catalog.rest.s3.endpoint http://rustfs:9000 spark.sql.catalog.rest.cache-enabled false # Configure Iceberg Hive catalog @@ -32,13 +32,13 @@ spark.sql.catalog.hive.type hive spark.sql.catalog.hive.uri thrift://hive:9083 spark.sql.catalog.hive.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.hive.warehouse s3://warehouse/hive/ -spark.sql.catalog.hive.s3.endpoint http://minio:9000 +spark.sql.catalog.hive.s3.endpoint http://rustfs:9000 # Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog spark.sql.catalog.spark_catalog.type hive spark.sql.catalog.spark_catalog.uri thrift://hive:9083 -spark.hadoop.fs.s3a.endpoint http://minio:9000 +spark.hadoop.fs.s3a.endpoint http://rustfs:9000 spark.sql.catalogImplementation hive spark.sql.warehouse.dir s3a://warehouse/hive/ diff --git a/mkdocs/docs/contributing.md b/mkdocs/docs/contributing.md index 0155907a65..025109346a 100644 --- a/mkdocs/docs/contributing.md +++ b/mkdocs/docs/contributing.md @@ -155,7 +155,7 @@ For Python, `pytest` is used a testing framework in combination with `coverage` make test ``` -By default, S3 and ADLS tests are ignored because that require minio and azurite to be running. +By default, S3 and ADLS tests are ignored because that require RustFS and azurite to be running. To run the S3 suite: ```bash @@ -257,7 +257,7 @@ This command spins up the full integration test infrastructure via Docker Compos - **Spark** (with Spark Connect) - **Iceberg REST Catalog** (using the [`apache/iceberg-rest-fixture`](https://hub.docker.com/r/apache/iceberg-rest-fixture) image) - **Hive Metastore** -- **S3-compatible object storage** (Minio) +- **S3-compatible object storage** (RustFS) **Spark Example Notebook** (`notebooks/spark_integration_example.ipynb`) is based on the [Spark Getting Started](https://iceberg.apache.org/docs/nightly/spark-getting-started/) guide. This notebook demonstrates how to work with PyIceberg alongside Spark, leveraging the Docker-based testing setup for a complete local development environment. From 21c577f0b388320ddc082eb16d1b3669092e0d6b Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Thu, 10 Sep 2026 22:03:19 +0000 Subject: [PATCH 2/2] Infra: Rename the object store service to `object-store` Feedback from @kevinjqliu --- dev/docker-compose-integration.yml | 24 ++++++++++++------------ dev/docker-compose.yml | 18 +++++++++--------- dev/hive/core-site.xml | 2 +- dev/run-s3.sh | 2 +- dev/spark/spark-defaults.conf | 6 +++--- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index b74daacc31..3eab22d9f7 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -28,7 +28,7 @@ services: depends_on: - rest - hive - - rustfs + - object-store ports: - 15002:15002 # Spark Connect - 4040:4040 # Spark UI @@ -39,7 +39,7 @@ services: links: - rest:rest - hive:hive - - rustfs:rustfs + - object-store:object-store healthcheck: test: ["CMD", "sh", "-c", "netstat -an | grep 15002 | grep LISTEN"] interval: 30s @@ -59,15 +59,15 @@ services: - AWS_REGION=us-east-1 - CATALOG_WAREHOUSE=s3://warehouse/ - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - - CATALOG_S3_ENDPOINT=http://rustfs:9000 + - CATALOG_S3_ENDPOINT=http://object-store:9000 - CATALOG_JDBC_STRICT__MODE=true - rustfs: + object-store: image: rustfs/rustfs:1.0.0-rc.5 - container_name: pyiceberg-rustfs + container_name: pyiceberg-object-store networks: iceberg_net: aliases: - - warehouse.rustfs + - warehouse.object-store ports: - 9001:9001 - 9000:9000 @@ -78,9 +78,9 @@ services: - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - RUSTFS_CONSOLE_ADDRESS=:9001 - # Resolves virtual-hosted-style `warehouse.rustfs` to the warehouse bucket. - # The port is required until rustfs/rustfs#7051 ships. - - RUSTFS_SERVER_DOMAINS=rustfs:9000 + # Resolves virtual-hosted-style `warehouse.object-store` to the warehouse + # bucket. The port is required until rustfs/rustfs#7051 ships. + - RUSTFS_SERVER_DOMAINS=object-store:9000 healthcheck: test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"] interval: 5s @@ -93,17 +93,17 @@ services: networks: iceberg_net: depends_on: - rustfs: + object-store: condition: service_healthy environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password - AWS_REGION=us-east-1 - - AWS_ENDPOINT_URL=http://rustfs:9000 + - AWS_ENDPOINT_URL=http://object-store:9000 entrypoint: /bin/sh command: - -c - - "aws s3 mb s3://warehouse" + - "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse" hive: image: pyiceberg-hive:latest build: diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 2db3572ad1..8ef82731e7 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -16,9 +16,12 @@ # under the License. services: - rustfs: + object-store: image: rustfs/rustfs:1.0.0-rc.5 - container_name: pyiceberg-rustfs + container_name: pyiceberg-object-store + ports: + - 9001:9001 + - 9000:9000 environment: - RUSTFS_ACCESS_KEY=admin - RUSTFS_SECRET_KEY=password @@ -26,9 +29,6 @@ services: - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - RUSTFS_CONSOLE_ADDRESS=:9001 - ports: - - 9001:9001 - - 9000:9000 healthcheck: test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"] interval: 5s @@ -36,16 +36,16 @@ services: retries: 10 start_period: 5s aws-cli: - depends_on: - rustfs: - condition: service_healthy image: amazon/aws-cli:2.36.40 container_name: pyiceberg-aws-cli + depends_on: + object-store: + condition: service_healthy environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password - AWS_REGION=us-east-1 - - AWS_ENDPOINT_URL=http://rustfs:9000 + - AWS_ENDPOINT_URL=http://object-store:9000 entrypoint: /bin/sh command: - -c diff --git a/dev/hive/core-site.xml b/dev/hive/core-site.xml index fea864f9d9..d17656f01a 100644 --- a/dev/hive/core-site.xml +++ b/dev/hive/core-site.xml @@ -32,7 +32,7 @@ fs.s3a.endpoint - http://rustfs:9000 + http://object-store:9000 fs.s3a.access.key diff --git a/dev/run-s3.sh b/dev/run-s3.sh index 7a37742890..359abd5b1e 100755 --- a/dev/run-s3.sh +++ b/dev/run-s3.sh @@ -20,7 +20,7 @@ set -ex -if [ $(docker ps -q --filter "name=pyiceberg-rustfs" --filter "health=healthy" ) ]; then +if [ $(docker ps -q --filter "name=pyiceberg-object-store" --filter "health=healthy" ) ]; then echo "S3 backend running" else docker compose -f dev/docker-compose.yml kill diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf index fcf40390ba..774155d209 100644 --- a/dev/spark/spark-defaults.conf +++ b/dev/spark/spark-defaults.conf @@ -23,7 +23,7 @@ spark.sql.catalog.rest.type rest spark.sql.catalog.rest.uri http://rest:8181 spark.sql.catalog.rest.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.rest.warehouse s3://warehouse/rest/ -spark.sql.catalog.rest.s3.endpoint http://rustfs:9000 +spark.sql.catalog.rest.s3.endpoint http://object-store:9000 spark.sql.catalog.rest.cache-enabled false # Configure Iceberg Hive catalog @@ -32,13 +32,13 @@ spark.sql.catalog.hive.type hive spark.sql.catalog.hive.uri thrift://hive:9083 spark.sql.catalog.hive.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.hive.warehouse s3://warehouse/hive/ -spark.sql.catalog.hive.s3.endpoint http://rustfs:9000 +spark.sql.catalog.hive.s3.endpoint http://object-store:9000 # Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog spark.sql.catalog.spark_catalog.type hive spark.sql.catalog.spark_catalog.uri thrift://hive:9083 -spark.hadoop.fs.s3a.endpoint http://rustfs:9000 +spark.hadoop.fs.s3a.endpoint http://object-store:9000 spark.sql.catalogImplementation hive spark.sql.warehouse.dir s3a://warehouse/hive/