From edec3d1f47c9a409395f4ad6e88fbc1b679602c8 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Mon, 16 Nov 2026 16:28:07 +0800 Subject: [PATCH 1/2] update img sync --- .github/workflows/build-docker.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index bf23aaa..6c860f1 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -26,9 +26,6 @@ env: # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. DOCKER_REGISTRY_USERNAME: ${{ vars.DOCKER_REGISTRY_USERNAME }} DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} - # used to sync image to mirror registry - DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }} - DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-dev' }} jobs: @@ -266,9 +263,10 @@ jobs: - env: AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }} + DOCKER_MIRROR_TARGETS: ${{ vars.DOCKER_MIRROR_TARGETS }} run: | source ./tool.sh printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \ - python /opt/utils/image-syncer/run_jobs.py --auth-file=/tmp/.github/workflows/auth.json + script-sync-image.sh --file-auth=/tmp/.github/workflows/auth.json From a48116e58459e6764b8a7dc5de60c9f00758c0c2 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Mon, 16 Nov 2026 17:22:42 +0800 Subject: [PATCH 2/2] debug sync --- .github/workflows/build-docker.yml | 1 - tool.sh | 36 ++++++++++++++++-------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 6c860f1..540b270 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -20,7 +20,6 @@ permissions: env: TZ: Asia/Shanghai - BUILDKIT_PROGRESS: "plain" # Full logs for CI build. REGISTRY_SRC: ${{ vars.REGISTRY_SRC || 'quay.io' }} # For BASE_NAMESPACE of images: where to pull base images from, docker.io or other source registry URL. REGISTRY_DST: ${{ vars.REGISTRY_DST || 'quay.io' }} # For tags of built images: where to push images to, docker.io or other destination registry URL. # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. diff --git a/tool.sh b/tool.sh index 43285ef..b091de0 100644 --- a/tool.sh +++ b/tool.sh @@ -13,20 +13,20 @@ else fi -CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/') - -# If on the main branch, image namespace will be same as CI_PROJECT_NAME's name space; -# else (not main branch), image namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE}. -[ "${CI_PROJECT_BRANCH}" = "main" ] && NAMESPACE_SUFFIX="" || NAMESPACE_SUFFIX="0${CI_PROJECT_SPACE}" ; -export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})${NAMESPACE_SUFFIX}" ; +# On main, use the repository owner's namespace; on other branches, append the fixed "0dev" suffix. +[ "${CI_PROJECT_BRANCH}" = "main" ] && NAMESPACE_SUFFIX="" || NAMESPACE_SUFFIX="0dev" ; +export CI_PROJECT_NAMESPACE="$(dirname "${CI_PROJECT_NAME}")${NAMESPACE_SUFFIX}" ; export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') -export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"quay.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') -export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_NAMESPACE_SRC=$(echo "${IMG_NAMESPACE_SRC:-$IMG_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_NAMESPACE_DST=$(echo "${IMG_NAMESPACE_DST:-$IMG_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"quay.io"}/${IMG_NAMESPACE_SRC}" | awk '{print tolower($0)}') +export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${IMG_NAMESPACE_SRC}" | awk '{print tolower($0)}') export TAG_SUFFIX="-$(git rev-parse --short HEAD)" echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" -echo "--------> DOCKER_IMG_NAMESPACE=${IMG_NAMESPACE}" +echo "--------> DOCKER_IMG_NAMESPACE_SRC=${IMG_NAMESPACE_SRC}" +echo "--------> DOCKER_IMG_NAMESPACE_DST=${IMG_NAMESPACE_DST}" echo "--------> DOCKER_IMG_PREFIX_SRC=${IMG_PREFIX_SRC}" echo "--------> DOCKER_IMG_PREFIX_DST=${IMG_PREFIX_DST}" echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}" @@ -34,7 +34,7 @@ echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}" build_image() { echo "$@" ; - IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)"; + local IMG=$1 TAG=$2 FILE=$3; shift 3; local VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX} WORKDIR="$(dirname "$FILE")"; BUILDKIT_PROGRESS=plain DOCKER_BUILDKIT=1 \ docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" \ --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" @@ -44,7 +44,7 @@ build_image() { build_image_no_tag() { echo "$@" ; - IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)"; + local IMG=$1 TAG=$2 FILE=$3; shift 3; local WORKDIR="$(dirname "$FILE")"; BUILDKIT_PROGRESS=plain DOCKER_BUILDKIT=1 \ docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" \ --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" @@ -52,14 +52,14 @@ build_image_no_tag() { } alias_image() { - IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; + local IMG_1=$1 TAG_1=$2 IMG_2=$3 TAG_2=$4; shift 4; local VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; docker tag "${IMG_PREFIX_DST}/${IMG_1}:${TAG_1}" "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" docker tag "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" "${IMG_PREFIX_DST}/${IMG_2}:${VER}" } push_image() { - KEYWORD="${1:-second}"; - docker image prune --force && docker images ; + local KEYWORD="${1:-second}" IMAGES IMG status; + docker image prune --force && docker images; IMAGES=$(docker images --format "{{.Repository}}\t{{.Tag}}\t{{.CreatedSince}}" | grep "${IMG_PREFIX_DST}" | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; [ -n "${IMAGES}" ] || { echo "!! No images matched keyword: ${KEYWORD}"; return 1; } echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_DST}" -u "$DOCKER_REGISTRY_USERNAME" --password-stdin ; @@ -72,9 +72,10 @@ push_image() { } clear_images() { - KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images built days ago - IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; - IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ; + local KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images built days ago + local IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; + local IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ; + local IMG status for IMG in $(echo "$IMGS_1 $IMGS_2" | tr " " "\n") ; do docker rmi "${IMG}" ; status=$?; echo "[${status}] image removed > ${IMG}"; @@ -84,6 +85,7 @@ clear_images() { remove_folder() { + local dir for dir in "$@"; do if [ -d "$dir" ]; then echo "Removing folder: $dir" ;