Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions .github/workflows/build-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,11 @@ permissions:

env:
TZ: Asia/Shanghai
BUILDKIT_PROGRESS: "plain" # Full logs for CI build.
REGISTRY_SRC: ${{ vars.REGISTRY_SRC || 'quay.io' }} # For BASE_NAMESPACE of images: where to pull base images from, docker.io or other source registry URL.
REGISTRY_DST: ${{ vars.REGISTRY_DST || 'quay.io' }} # For tags of built images: where to push images to, docker.io or other destination registry URL.
# DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets.
DOCKER_REGISTRY_USERNAME: ${{ vars.DOCKER_REGISTRY_USERNAME }}
DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
# used to sync image to mirror registry
DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }}
DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }}
CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-dev' }}

jobs:
Expand Down Expand Up @@ -266,9 +262,10 @@ jobs:
- env:
AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }}
DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }}
DOCKER_MIRROR_TARGETS: ${{ vars.DOCKER_MIRROR_TARGETS }}
run: |
source ./tool.sh
printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows
printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env
docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \
python /opt/utils/image-syncer/run_jobs.py --auth-file=/tmp/.github/workflows/auth.json
script-sync-image.sh --file-auth=/tmp/.github/workflows/auth.json
36 changes: 19 additions & 17 deletions tool.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,28 +13,28 @@ else
fi


CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/')

# If on the main branch, image namespace will be same as CI_PROJECT_NAME's name space;
# else (not main branch), image namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE}.
[ "${CI_PROJECT_BRANCH}" = "main" ] && NAMESPACE_SUFFIX="" || NAMESPACE_SUFFIX="0${CI_PROJECT_SPACE}" ;
export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})${NAMESPACE_SUFFIX}" ;
# On main, use the repository owner's namespace; on other branches, append the fixed "0dev" suffix.
[ "${CI_PROJECT_BRANCH}" = "main" ] && NAMESPACE_SUFFIX="" || NAMESPACE_SUFFIX="0dev" ;
export CI_PROJECT_NAMESPACE="$(dirname "${CI_PROJECT_NAME}")${NAMESPACE_SUFFIX}" ;

export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}')
export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"quay.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}')
export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}')
export IMG_NAMESPACE_SRC=$(echo "${IMG_NAMESPACE_SRC:-$IMG_NAMESPACE}" | awk '{print tolower($0)}')
export IMG_NAMESPACE_DST=$(echo "${IMG_NAMESPACE_DST:-$IMG_NAMESPACE}" | awk '{print tolower($0)}')
export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"quay.io"}/${IMG_NAMESPACE_SRC}" | awk '{print tolower($0)}')
export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${IMG_NAMESPACE_SRC}" | awk '{print tolower($0)}')
export TAG_SUFFIX="-$(git rev-parse --short HEAD)"

echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}"
echo "--------> DOCKER_IMG_NAMESPACE=${IMG_NAMESPACE}"
echo "--------> DOCKER_IMG_NAMESPACE_SRC=${IMG_NAMESPACE_SRC}"
echo "--------> DOCKER_IMG_NAMESPACE_DST=${IMG_NAMESPACE_DST}"
echo "--------> DOCKER_IMG_PREFIX_SRC=${IMG_PREFIX_SRC}"
echo "--------> DOCKER_IMG_PREFIX_DST=${IMG_PREFIX_DST}"
echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}"


build_image() {
echo "$@" ;
IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)";
local IMG=$1 TAG=$2 FILE=$3; shift 3; local VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX} WORKDIR="$(dirname "$FILE")";
BUILDKIT_PROGRESS=plain DOCKER_BUILDKIT=1 \
docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" \
--build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}"
Expand All @@ -44,22 +44,22 @@ build_image() {

build_image_no_tag() {
echo "$@" ;
IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)";
local IMG=$1 TAG=$2 FILE=$3; shift 3; local WORKDIR="$(dirname "$FILE")";
BUILDKIT_PROGRESS=plain DOCKER_BUILDKIT=1 \
docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" \
--build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}"
echo "${IMG_PREFIX_DST}/${IMG}:${TAG}"
}

alias_image() {
IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX};
local IMG_1=$1 TAG_1=$2 IMG_2=$3 TAG_2=$4; shift 4; local VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX};
docker tag "${IMG_PREFIX_DST}/${IMG_1}:${TAG_1}" "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}"
docker tag "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" "${IMG_PREFIX_DST}/${IMG_2}:${VER}"
}

push_image() {
KEYWORD="${1:-second}";
docker image prune --force && docker images ;
local KEYWORD="${1:-second}" IMAGES IMG status;
docker image prune --force && docker images;
IMAGES=$(docker images --format "{{.Repository}}\t{{.Tag}}\t{{.CreatedSince}}" | grep "${IMG_PREFIX_DST}" | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ;
[ -n "${IMAGES}" ] || { echo "!! No images matched keyword: ${KEYWORD}"; return 1; }
echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_DST}" -u "$DOCKER_REGISTRY_USERNAME" --password-stdin ;
Expand All @@ -72,9 +72,10 @@ push_image() {
}

clear_images() {
KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images built days ago
IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ;
IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ;
local KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images built days ago
local IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ;
local IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ;
local IMG status

for IMG in $(echo "$IMGS_1 $IMGS_2" | tr " " "\n") ; do
docker rmi "${IMG}" ; status=$?; echo "[${status}] image removed > ${IMG}";
Expand All @@ -84,6 +85,7 @@ clear_images() {


remove_folder() {
local dir
for dir in "$@"; do
if [ -d "$dir" ]; then
echo "Removing folder: $dir" ;
Expand Down
Loading