Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions .github/workflows/build-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,17 @@ on:
pull_request:
branches: [ "main" ]
paths-ignore: [ "*.md" ]

workflow_dispatch: # Allows you to run this workflow manually from the Actions tab

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

permissions:
contents: read

env:
BUILDKIT_PROGRESS: "plain" # Full logs for CI build.
TZ: Asia/Shanghai
REGISTRY_SRC: ${{ vars.REGISTRY_SRC || 'quay.io' }} # For BASE_NAMESPACE of images: where to pull base images from, docker.io or other source registry URL.
REGISTRY_DST: ${{ vars.REGISTRY_DST || 'quay.io' }} # For tags of built images: where to push images to, docker.io or other destination registry URL.
# DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets.
Expand All @@ -24,32 +26,38 @@ env:
# used to sync image to mirror registry
DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }}
DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }}
CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-container' }}
CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-container-kit' }}

jobs:
job-k3s-ctk:
name: 'k3s-ctk'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
- run: |
source ./tool.sh && build_image k3s-ctk latest docker_kube/k3s-ctk.Dockerfile && push_image
source ./tool.sh && K8S_VERSION=1.37.1
build_image_no_tag k3s-ctk latest docker_kube/k3s-ctk.Dockerfile --build-arg K8S_VERSION=${K8S_VERSION}
alias_image k3s-ctk latest k3s-ctk ${K8S_VERSION}
push_image ${K8S_VERSION}

job-k0s-ctk:
name: 'k0s-ctk'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
- run: |
source ./tool.sh && build_image k0s-ctk latest docker_kube/k0s-ctk.Dockerfile && push_image
source ./tool.sh && K8S_VERSION=1.37.1
build_image_no_tag k0s-ctk latest docker_kube/k0s-ctk.Dockerfile --build-arg K8S_VERSION=${K8S_VERSION}
alias_image k0s-ctk latest k0s-ctk ${K8S_VERSION}
push_image ${K8S_VERSION}


## Sync all images in this build (listed by "names") to mirror registry.
sync_images:
needs: ["job-k3s-ctk", "job-k0s-ctk"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
- env:
AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }}
DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }}
Expand Down
80 changes: 58 additions & 22 deletions docker_kube/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ The tools and files included in each container image:
| Component Role | `k3s-ctk` Image | `k0s-ctk` Image | Description |
| :--- | :--- | :--- | :--- |
| **Kubernetes Engine** | `k3s` | `k0s` | The core Kubernetes distribution binary. |
| **Air-gap Image Bundle** | `k3s-airgap-images-*.tar.zst` | `k0s-airgap-bundle-*` | System container images bundle for offline deployment. |
| **Air-gap Image Bundle** | `k3s-airgap-images-*.tar.zst` | `k0s-airgap-bundle-*.tar` | System container images bundle for offline deployment. |
| **Kubernetes CLI** | `kubectl` | `kubectl` | Command-line tool for control plane operations. |
| **Package Manager** | `helm` | `helm` | Tool for managing Kubernetes pre-packaged charts. |
| **CRI Adapter** | `cri-dockerd` | *N/A* | Adapter to use Docker as the container runtime (embedded in K0s). |
| **Official Installer** | `script-get-k3s-io.sh` | *N/A* | Official installation script (embedded in K0s). |
| **CRI Adapter** | (embedded in k3s) | `cri-dockerd` | Adapter to use Docker as the container runtime. |
| **Official Installer** | `script-get-k3s-io.sh` | (embedded in k0s) | Official installation script. |
| **Setup Helper Script** | `script-setup-k3s.sh` | `script-setup-k0s.sh` | Scripts containing functions for configuration and setup. |

---
Expand All @@ -45,28 +45,41 @@ Import the K3s system images into your Docker daemon:
```bash
cd /opt/k3s
zstd -cd ./k3s-airgap-images-${ARCH:-amd64}.tar.zst | docker load
```

#### 3. Configure and start cri-dockerd
```bash
# Generate systemd socket and service unit files
source ./script-setup-k3s.sh && create_cri_dockerd_unit_files
sudo rm -f /opt/k3s/k3s-*.tar.zst

# Enable and start the service
sudo systemctl daemon-reload
sudo systemctl enable --now cri-docker.socket cri-docker.service
# (Optional) Link binaries to PATH
sudo ln -sf /opt/k3s/k* /opt/k3s/helm /usr/local/bin/
```

#### 4. Run the offline installer
#### 3. Run the offline installer
```bash
INSTALL_K3S_SKIP_DOWNLOAD=true \
INSTALL_K3S_BIN_DIR=/opt/k3s \
./script-get-k3s-io.sh --docker --data-dir /data/storage/data-k3s
```

#### 5. Link binaries to PATH (Optional)
#### 4. (Optional) Manually start k3s without systemd
For environment like Github Codespace, you might want to start k3s without systemd:
```bash
sudo ln -sf /opt/k3s/k3s* /opt/k3s/kubectl /opt/k3s/cri-dockerd /usr/local/bin/
sudo k3s server \
--docker \
--container-runtime-endpoint unix:///var/run/docker.sock
```

#### 5. Access the k3s cluster with kubectl

Copy the k3s default kube config file `/etc/rancher/k3s/k3s.yaml` for current user and use kubectl.

```bash
mkdir -pv ~/.kube
sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
sudo chown "$(id -u):$(id -g)" ~/.kube/config
chmod 600 ~/.kube/config

kubectl cluster-info
kubectl get nodes -o wide
kubectl get pods -A
kubectl get services -A
```

---
Expand All @@ -84,25 +97,48 @@ K0s loads offline images from `/var/lib/k0s/images/` by default.
```bash
cd /opt/k0s

# Copy binaries to PATH
sudo cp ./k0s ./kubectl ./helm /usr/local/bin/

# Copy the air-gap bundle to the designated directory
# Load the docker images in the air-gap bundle
sudo mkdir -p /var/lib/k0s/images/
sudo cp ./k0s-airgap-bundle-* /var/lib/k0s/images/
sudo cat ./k0s-airgap-bundle-* | docker load
sudo rm -f ./k0s-airgap-bundle-*

# (Optional) Link binaries to PATH
sudo ln -sf /opt/k0s/k* /opt/k0s/helm /usr/local/bin/
```

#### 3. Install and start K0s service
```bash
# (Optional) Setup cri-dockerd for k0s to use docker daemon instead of containerd as runtime
source ./script-setup-k0s.sh && create_cri_dockerd_unit_files

# Install as a single-node controller + worker
sudo k0s install controller --single --enable-worker

# Start the service
sudo k0s start
```

#### 4. Verify installation
#### 4. (Optional) Manually start k0s without systemd
```bash
sudo /opt/k0s/cri-dockerd \
--docker-endpoint=unix:///var/run/docker.sock \
--container-runtime-endpoint=unix:///run/cri-dockerd.sock

sudo k0s controller --single --enable-worker \
--cri-socket=remote:unix:///run/cri-dockerd.sock -v
```

#### 5. Verify installation
```bash
sudo k0s status
sudo k0s kubectl get nodes

mkdir -pv ~/.kube
sudo k0s kubeconfig admin > ~/.kube/config
sudo chown "$(id -u):$(id -g)" ~/.kube/config
chmod 600 ~/.kube/config

kubectl cluster-info
kubectl get nodes -o wide
kubectl get pods -A
kubectl get services -A
```
9 changes: 6 additions & 3 deletions docker_kube/k0s-ctk.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,19 @@ ARG BASE_NAMESPACE
ARG BASE_IMG="atom"
FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} AS runner

ARG K8S_VERSION

LABEL maintainer="haobibo@gmail.com"

COPY work /opt/utils/

RUN set -eux && chmod +x /opt/utils/*.sh \
&& source /opt/utils/script-setup-k8s-common.sh && setup_verify_arch && setup_kubectl && setup_helm && setup_k9s \
&& source /opt/utils/script-setup-k0s.sh && setup_k0s && setup_k0s_pack \
&& source /opt/utils/script-setup-k8s-common.sh && setup_verify_arch && setup_helm && setup_k9s \
&& source /opt/utils/script-setup-k0s.sh \
&& setup_k0s "${K8S_VERSION:+${K8S_VERSION}+k0s.0}" && setup_k0s_pack "${K8S_VERSION:+${K8S_VERSION}+k0s.0}" && setup_kubectl "${K8S_VERSION}" \
## zstd -T0 --ultra -22 --rm /opt/k0s/*.tar \ ## skip due to very low compression rate
&& mv /opt/k8s/* /opt/k0s/ \
&& mv /opt/utils/script-setup-k0s.sh /opt/k0s/ \

&& rm -rf /opt/k8s && ls -alh /opt/*


Expand Down
11 changes: 7 additions & 4 deletions docker_kube/k3s-ctk.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,18 @@ ARG BASE_NAMESPACE
ARG BASE_IMG="atom"
FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} AS runner

ARG K8S_VERSION

LABEL maintainer="haobibo@gmail.com"

COPY work /opt/utils/

RUN set -eux && chmod +x /opt/utils/*.sh \
&& source /opt/utils/script-setup-k8s-common.sh && setup_verify_arch && setup_kubectl && setup_helm && setup_k9s \
&& source /opt/utils/script-setup-k3s.sh && setup_k3s && setup_cri_dockerd && setup_k3s_pack \
&& mv /opt/k8s/* /opt/k3s/ \
&& mv /opt/utils/script-setup-k3s.sh /opt/utils/script-get-k3s-io.sh /opt/k3s/ \
&& source /opt/utils/script-setup-k8s-common.sh && setup_verify_arch && setup_helm && setup_k9s \
&& source /opt/utils/script-setup-k3s.sh \
&& setup_k3s "${K8S_VERSION}" && setup_k3s_pack "${K8S_VERSION}" && setup_kubectl "${K8S_VERSION}" \
&& mv /opt/k8s/* /opt/k3s/ \
&& mv /opt/utils/script-setup-k3s.sh /opt/utils/script-get-k3s-io.sh /opt/k3s/ \
&& rm -rf /opt/k8s && ls -alh /opt/*


Expand Down
13 changes: 8 additions & 5 deletions docker_kube/work/script-get-k3s-io.sh
Original file line number Diff line number Diff line change
Expand Up @@ -606,13 +606,16 @@ setup_selinux() {
rpm_target=sle
rpm_site_infix=microos
package_installer=zypper
if [ "${ID_LIKE:-}" = suse ] && ( [ "${VARIANT_ID:-}" = sle-micro ] || [ "${ID:-}" = sle-micro ] ); then
rpm_target=sle

# SleMicro 6.2 and SLEs 16 uses the same ID and same VERSION_ID in case the ID and VERSION_ID changes we will catch SLEMicro 6.2 using VARIANT_ID transactional
if [ "${ID:-}" = sles ] && [ -n "${VERSION_ID}" ] && [ "${VERSION_ID%%.*}" -ge 16 ]; then
rpm_site_infix=slemicro
elif [ "${VARIANT_ID:-}" = sle-micro ] || [ "${VARIANT_ID:-}" = transactional ] || [ "${ID:-}" = sle-micro ] || [ "${ID:-}" = sl-micro ] || [ "${ID:-}" = sle-micro-rancher ]; then
rpm_site_infix=slemicro
package_installer=zypper
fi
# cover any atomic fedora flavors using rpm-ostree
elif { [ "${ID:-}" = fedora ] || [ "${ID_LIKE:-}" = fedora ]; } && [ -n "${OSTREE_VERSION:-}" ]; then

# cover coreos derivatives like flatcar, plus any atomic fedora flavors using rpm-ostree
elif [ "${ID_LIKE:-}" = coreos ] || { { [ "${ID:-}" = fedora ] || [ "${ID_LIKE:-}" = fedora ]; } && [ -n "${OSTREE_VERSION:-}" ]; }; then
rpm_target=coreos
rpm_site_infix=coreos
package_installer=rpm-ostree
Expand Down
92 changes: 87 additions & 5 deletions docker_kube/work/script-setup-k0s.sh
Original file line number Diff line number Diff line change
@@ -1,17 +1,99 @@
# k0s Air-gap Install: https://docs.k0sproject.io/head/airgap-install/

setup_k0s() {
VER_K0S=$(curl -sL https://docs.k0sproject.io/stable.txt) \
&& URL_K0S="https://github.com/k0sproject/k0s/releases/download/${VER_K0S}/k0s-${VER_K0S}-${ARCH}" \
local VER_K0S_REQ="${1:-}" VER_K0S VER_K0S_URL URL_K0S
VER_K0S="${VER_K0S_REQ#v}"
if [ -z "${VER_K0S}" ]; then
VER_K0S=$(curl -sL https://docs.k0sproject.io/stable.txt)
VER_K0S="${VER_K0S#v}"
fi
[ -n "${VER_K0S}" ] || return 1
VER_K0S="v${VER_K0S}"
VER_K0S_URL="${VER_K0S//+/%2B}"
URL_K0S="https://github.com/k0sproject/k0s/releases/download/${VER_K0S_URL}/k0s-${VER_K0S_URL}-${ARCH}" \
&& echo "Downloading k0s version ${VER_K0S} from ${URL_K0S}" \
&& mkdir -pv /opt/k0s && curl -L -o /opt/k0s/k0s ${URL_K0S} \
&& chmod +x /opt/k0s/k0s
/opt/k0s/k0s version
}

setup_k0s_pack() {
VER_K0S=$(curl -sL https://docs.k0sproject.io/stable.txt) \
&& URL_K0S_IMG="https://github.com/k0sproject/k0s/releases/download/${VER_K0S}/k0s-airgap-bundle-${VER_K0S}-${ARCH}" \
local VER_K0S_REQ="${1:-}" VER_K0S VER_K0S_URL URL_K0S_IMG
VER_K0S="${VER_K0S_REQ#v}"
if [ -z "${VER_K0S}" ]; then
VER_K0S=$(curl -sL https://docs.k0sproject.io/stable.txt)
VER_K0S="${VER_K0S#v}"
fi
[ -n "${VER_K0S}" ] || return 1
VER_K0S="v${VER_K0S}"
VER_K0S_URL="${VER_K0S//+/%2B}"
URL_K0S_IMG="https://github.com/k0sproject/k0s/releases/download/${VER_K0S_URL}/k0s-airgap-bundle-${VER_K0S_URL}-linux-${ARCH}.tar" \
&& echo "Downloading k0s airgap images version ${VER_K0S} from ${URL_K0S_IMG}" \
&& curl -L -o "/opt/k0s/k0s-airgap-bundle-${VER_K0S}-${ARCH}" ${URL_K0S_IMG}
&& curl -L -o "/opt/k0s/k0s-airgap-bundle-${VER_K0S}-linux-${ARCH}.tar" "${URL_K0S_IMG}"
}


setup_cri_dockerd() {
# Install the latest release: https://mirantis.github.io/cri-dockerd/usage/install-manually/
VER_CRI_DOCKERD=$(curl -sL https://github.com/Mirantis/cri-dockerd/releases.atom | grep 'releases/tag/v' | grep -v 'rc' | head -1 | grep -Po '\d[\d.]+' ) \
&& URL_CRI_DOCKERD="https://github.com/Mirantis/cri-dockerd/releases/download/v$VER_CRI_DOCKERD/cri-dockerd-$VER_CRI_DOCKERD.$ARCH.tgz" \
&& echo "Downloading cri-dockerd version ${VER_CRI_DOCKERD} from: ${URL_CRI_DOCKERD}" \
&& mkdir -pv /opt/k0s && curl -L -o /tmp/cri-dockerd.tgz $URL_CRI_DOCKERD \
&& tar -xzvf /tmp/cri-dockerd.tgz -C /opt/k0s/ --strip-components=1 cri-dockerd/cri-dockerd \
&& chmod +x /opt/k0s/cri-dockerd
/opt/k0s/cri-dockerd --version
}


create_cri_dockerd_unit_files() {
local SYSTEMD_DIR="/etc/systemd/system"
local FILE_SOCKET="${SYSTEMD_DIR}/cri-docker.socket"
local FILE_SERVICE="${SYSTEMD_DIR}/cri-docker.service"

echo "Creating ${FILE_SOCKET} and ${FILE_SERVICE} ..."

sudo tee "${FILE_SOCKET}" >/dev/null <<'EOF'
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=/var/run/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF

sudo tee "${FILE_SERVICE}" >/dev/null <<'EOF'
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify
ExecStart=/opt/k0s/cri-dockerd --container-runtime-endpoint fd://
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not support it.
TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target
EOF
}
11 changes: 0 additions & 11 deletions docker_kube/work/script-setup-k3d.sh

This file was deleted.

Loading
Loading