Infra: Replace MinIO with RustFS

Makefile

@@ -129,7 +129,7 @@ test-integration-rebuild: ## Rebuild integration Docker services from scratch
 	docker compose -f dev/docker-compose-integration.yml build --no-cache
 
 test-s3: ## Run tests marked with @pytest.mark.s3
-	sh ./dev/run-minio.sh
+	sh ./dev/run-s3.sh
 	$(TEST_RUNNER) pytest tests/ -m s3 $(PYTEST_ARGS)
 
 test-adls: ## Run tests marked with @pytest.mark.adls

dev/docker-compose-integration.yml

@@ -28,7 +28,7 @@ services:
     depends_on:
       - rest
       - hive
-      - minio
+      - object-store
     ports:
       - 15002:15002 # Spark Connect
       - 4040:4040 # Spark UI
@@ -39,7 +39,7 @@ services:
     links:
       - rest:rest
       - hive:hive
-      - minio:minio
+      - object-store:object-store
     healthcheck:
       test: ["CMD", "sh", "-c", "netstat -an | grep 15002 | grep LISTEN"]
       interval: 30s
@@ -59,41 +59,51 @@ services:
       - AWS_REGION=us-east-1
       - CATALOG_WAREHOUSE=s3://warehouse/
       - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
-      - CATALOG_S3_ENDPOINT=http://minio:9000
+      - CATALOG_S3_ENDPOINT=http://object-store:9000
       - CATALOG_JDBC_STRICT__MODE=true
-  minio:
-    image: minio/minio
-    container_name: pyiceberg-minio
+  object-store:
+    image: rustfs/rustfs:1.0.0-rc.5
+    container_name: pyiceberg-object-store
     networks:
       iceberg_net:
         aliases:
-          - warehouse.minio
+          - warehouse.object-store
     ports:
       - 9001:9001
       - 9000:9000
     environment:
-      - MINIO_ROOT_USER=admin
-      - MINIO_ROOT_PASSWORD=password
-      - MINIO_DOMAIN=minio
-    command: ["server", "/data", "--console-address", ":9001"]
-  mc:
-    image: minio/mc
-    container_name: pyiceberg-mc
+      - RUSTFS_ACCESS_KEY=admin
+      - RUSTFS_SECRET_KEY=password
+      - RUSTFS_VOLUMES=/data
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_CONSOLE_ADDRESS=:9001
+      # Resolves virtual-hosted-style `warehouse.object-store` to the warehouse
+      # bucket. The port is required until rustfs/rustfs#7051 ships.
+      - RUSTFS_SERVER_DOMAINS=object-store:9000
+    healthcheck:
+      test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 5s
+  aws-cli:
+    image: amazon/aws-cli:2.36.40
+    container_name: pyiceberg-aws-cli
     networks:
       iceberg_net:
     depends_on:
-      - minio
+      object-store:
+        condition: service_healthy
     environment:
       - AWS_ACCESS_KEY_ID=admin
       - AWS_SECRET_ACCESS_KEY=password
       - AWS_REGION=us-east-1
-    entrypoint: >
-      /bin/sh -c "
-      until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
-      /usr/bin/mc mb minio/warehouse;
-      /usr/bin/mc policy set public minio/warehouse;
-      tail -f /dev/null
-      "
+      - AWS_ENDPOINT_URL=http://object-store:9000
+    entrypoint: /bin/sh
+    command:
+      - -c
+      - "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse"
   hive:
     image: pyiceberg-hive:latest
     build:

dev/docker-compose.yml

@@ -16,31 +16,37 @@
 # under the License.
 
 services:
-  minio:
-    image: minio/minio
-    container_name: pyiceberg-minio
-    environment:
-      - MINIO_ROOT_USER=admin
-      - MINIO_ROOT_PASSWORD=password
-      - MINIO_DOMAIN=minio
+  object-store:
+    image: rustfs/rustfs:1.0.0-rc.5
+    container_name: pyiceberg-object-store
     ports:
       - 9001:9001
       - 9000:9000
-    command: ["server", "/data", "--console-address", ":9001"]
-  mc:
+    environment:
+      - RUSTFS_ACCESS_KEY=admin
+      - RUSTFS_SECRET_KEY=password
+      - RUSTFS_VOLUMES=/data
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_CONSOLE_ADDRESS=:9001
+    healthcheck:
+      test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 5s
+  aws-cli:
+    image: amazon/aws-cli:2.36.40
+    container_name: pyiceberg-aws-cli
     depends_on:
-      - minio
-    image: minio/mc
-    container_name: pyiceberg-mc
+      object-store:
+        condition: service_healthy
     environment:
       - AWS_ACCESS_KEY_ID=admin
       - AWS_SECRET_ACCESS_KEY=password
       - AWS_REGION=us-east-1
-    entrypoint: >
-      /bin/sh -c "
-      until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
-      /usr/bin/mc rm -r --force minio/warehouse;
-      /usr/bin/mc mb minio/warehouse;
-      /usr/bin/mc policy set public minio/warehouse;
-      exit 0;
-      "
+      - AWS_ENDPOINT_URL=http://object-store:9000
+    entrypoint: /bin/sh
+    command:
+      - -c
+      - "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse"

dev/hive/core-site.xml

@@ -32,7 +32,7 @@
     </property>
     <property>
       <name>fs.s3a.endpoint</name>
-      <value>http://minio:9000</value>
+      <value>http://object-store:9000</value>
     </property>
     <property>
       <name>fs.s3a.access.key</name>

dev/run-minio.sh → dev/run-s3.sh

@@ -20,14 +20,9 @@
 
 set -ex
 
-if [ $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]; then
-    echo "Minio backend running"
+if [ $(docker ps -q --filter "name=pyiceberg-object-store" --filter "health=healthy" ) ]; then
+    echo "S3 backend running"
 else
     docker compose -f dev/docker-compose.yml kill
-    docker compose -f dev/docker-compose.yml up -d
-    while [ -z $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]
-    do
-      echo "Waiting for Minio"
-      sleep 1
-    done
+    docker compose -f dev/docker-compose.yml up -d --wait
 fi

dev/spark/spark-defaults.conf

@@ -23,7 +23,7 @@ spark.sql.catalog.rest.type            rest
 spark.sql.catalog.rest.uri             http://rest:8181
 spark.sql.catalog.rest.io-impl         org.apache.iceberg.aws.s3.S3FileIO
 spark.sql.catalog.rest.warehouse       s3://warehouse/rest/
-spark.sql.catalog.rest.s3.endpoint     http://minio:9000
+spark.sql.catalog.rest.s3.endpoint     http://object-store:9000
 spark.sql.catalog.rest.cache-enabled   false
 
 # Configure Iceberg Hive catalog
@@ -32,13 +32,13 @@ spark.sql.catalog.hive.type            hive
 spark.sql.catalog.hive.uri             thrift://hive:9083
 spark.sql.catalog.hive.io-impl         org.apache.iceberg.aws.s3.S3FileIO
 spark.sql.catalog.hive.warehouse       s3://warehouse/hive/
-spark.sql.catalog.hive.s3.endpoint     http://minio:9000
+spark.sql.catalog.hive.s3.endpoint     http://object-store:9000
 
 # Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore
 spark.sql.catalog.spark_catalog        org.apache.iceberg.spark.SparkSessionCatalog
 spark.sql.catalog.spark_catalog.type   hive
 spark.sql.catalog.spark_catalog.uri    thrift://hive:9083
-spark.hadoop.fs.s3a.endpoint           http://minio:9000
+spark.hadoop.fs.s3a.endpoint           http://object-store:9000
 spark.sql.catalogImplementation        hive
 spark.sql.warehouse.dir                s3a://warehouse/hive/
 

mkdocs/docs/contributing.md

@@ -155,7 +155,7 @@ For Python, `pytest` is used a testing framework in combination with `coverage`
 make test
 ```
 
-By default, S3 and ADLS tests are ignored because that require minio and azurite to be running.
+By default, S3 and ADLS tests are ignored because that require RustFS and azurite to be running.
 To run the S3 suite:
 
 ```bash
@@ -257,7 +257,7 @@ This command spins up the full integration test infrastructure via Docker Compos
 - **Spark** (with Spark Connect)
 - **Iceberg REST Catalog** (using the [`apache/iceberg-rest-fixture`](https://hub.docker.com/r/apache/iceberg-rest-fixture) image)
 - **Hive Metastore**
-- **S3-compatible object storage** (Minio)
+- **S3-compatible object storage** (RustFS)
 
 **Spark Example Notebook** (`notebooks/spark_integration_example.ipynb`) is based on the [Spark Getting Started](https://iceberg.apache.org/docs/nightly/spark-getting-started/) guide. This notebook demonstrates how to work with PyIceberg alongside Spark, leveraging the Docker-based testing setup for a complete local development environment.