From 435229255bb0b865dd7696860c27d2b13a7ea3d1 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Sat, 14 Nov 2026 20:10:52 +0000 Subject: [PATCH 1/8] update img sync --- .github/workflows/build-docker-db.yml | 2 +- .github/workflows/build-docker-gpu.yml | 2 +- .github/workflows/build-docker.yml | 4 +- docker_atom/work/script-setup-core.sh | 2 +- docker_docker_kit/README.md | 6 +- docker_docker_kit/docker-kit.Dockerfile | 13 +- .../work/image-syncer/run_jobs.py | 60 ------ .../work/image-syncer/run_sync.py | 97 --------- docker_docker_kit/work/script-sync-image.sh | 199 ++++++++++++++++++ 9 files changed, 212 insertions(+), 173 deletions(-) delete mode 100644 docker_docker_kit/work/image-syncer/run_jobs.py delete mode 100644 docker_docker_kit/work/image-syncer/run_sync.py create mode 100644 docker_docker_kit/work/script-sync-image.sh diff --git a/.github/workflows/build-docker-db.yml b/.github/workflows/build-docker-db.yml index 82ed7bb9..e706d75f 100644 --- a/.github/workflows/build-docker-db.yml +++ b/.github/workflows/build-docker-db.yml @@ -61,5 +61,5 @@ jobs: printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \ - python /opt/utils/image-syncer/run_jobs.py --auth-file=/tmp/.github/workflows/auth.json \ + sh /opt/utils/script-sync-image.sh --auth-file=/tmp/.github/workflows/auth.json \ --workflow-file=".github/workflows/build-docker-db.yml" diff --git a/.github/workflows/build-docker-gpu.yml b/.github/workflows/build-docker-gpu.yml index bf36f6c1..1bda79d0 100644 --- a/.github/workflows/build-docker-gpu.yml +++ b/.github/workflows/build-docker-gpu.yml @@ -166,5 +166,5 @@ jobs: printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \ - python /opt/utils/image-syncer/run_jobs.py --auth-file=/tmp/.github/workflows/auth.json \ + sh /opt/utils/script-sync-image.sh --auth-file=/tmp/.github/workflows/auth.json \ --workflow-file=".github/workflows/build-docker-gpu.yml" diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index f2ebc435..7739c983 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -234,7 +234,7 @@ jobs: printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \ - python /opt/utils/image-syncer/run_jobs.py --auth-file=/tmp/.github/workflows/auth.json + sh /opt/utils/script-sync-image.sh --auth-file=/tmp/.github/workflows/auth.json ## Sync all images in this build (listed by "names") to mirror registry. job-sync-images: @@ -250,4 +250,4 @@ jobs: printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \ - python /opt/utils/image-syncer/run_jobs.py --auth-file=/tmp/.github/workflows/auth.json + sh /opt/utils/script-sync-image.sh --auth-file=/tmp/.github/workflows/auth.json diff --git a/docker_atom/work/script-setup-core.sh b/docker_atom/work/script-setup-core.sh index 846f1c53..12460ef9 100755 --- a/docker_atom/work/script-setup-core.sh +++ b/docker_atom/work/script-setup-core.sh @@ -419,7 +419,7 @@ setup_yq() { && echo "Installing yq v${VER_YQ} for arch ${ARCH} from: ${URL_YQ}" \ && curl -fSL "${URL_YQ}" -o /tmp/yq \ && install -m 0755 -D /tmp/yq /opt/bin/yq \ - && ln -sf /opt/bin/yq /usr/bin/yq \ + && sudo ln -sf /opt/bin/yq /usr/bin/yq \ && rm -f /tmp/yq ; type yq && echo "@ Installed yq: $(yq --version)" ; } diff --git a/docker_docker_kit/README.md b/docker_docker_kit/README.md index 90b213d3..450cd3f3 100644 --- a/docker_docker_kit/README.md +++ b/docker_docker_kit/README.md @@ -4,7 +4,7 @@ In this docker image (DockerKit), the following tools are included: - [docker-compose](https://github.com/docker/compose): tool for running multi-container applications on Docker. -- [image-syncer](https://github.com/AliyunContainerService/image-syncer): along with some Python scripts used to sync docker images between docker registries. +- [image-syncer](https://github.com/AliyunContainerService/image-syncer): used to sync docker images between docker registries. ## Image Syncer @@ -20,10 +20,10 @@ docker run --rm \ -e DOCKER_REGISTRY_USERNAME='' \ -e DOCKER_MIRROR_REGISTRY_USERNAME='' \ -e DOCKER_MIRROR_REGISTRY_PASSWORD='' \ - labnow/docker-kit python /opt/utils/image-syncer/run_sync.py library/ubuntu --source-registry='quay.io' --target-registry='docker.io' + labnow/docker-kit sh /opt/utils/script-sync-image.sh library/ubuntu --source-registry='quay.io' --target-registry='docker.io' ``` -Fill the environment variables above, and more details can be found in code: `./work/image-syncer/run_sync.py`. +Fill the environment variables above, and more details can be found in `./work/script-sync-image.sh`. ### Sync images in batch diff --git a/docker_docker_kit/docker-kit.Dockerfile b/docker_docker_kit/docker-kit.Dockerfile index 9fe4abed..db9754e5 100644 --- a/docker_docker_kit/docker-kit.Dockerfile +++ b/docker_docker_kit/docker-kit.Dockerfile @@ -1,7 +1,7 @@ # Distributed under the terms of the Modified BSD License. ARG BASE_NAMESPACE -ARG BASE_IMG="base" +ARG BASE_IMG="atom" FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} LABEL maintainer="postmaster@labnow.ai" @@ -10,11 +10,8 @@ ENV XDG_RUNTIME_DIR="/tmp" COPY work /opt/utils/ RUN set -eux \ - && source /opt/utils/script-setup-core.sh \ - && setup_yq \ - && source /opt/utils/script-setup-docker.sh \ - && setup_docker_compose && setup_docker_syncer \ - && pip install -U PyYaml \ - && ln -sf /usr/bin/image-syncer /opt/utils/image-syncer/ \ - # Clean up and display components version information... + && chmod +x /opt/utils/*.sh \ + && source /opt/utils/script-setup-core.sh && setup_yq \ + && source /opt/utils/script-setup-docker.sh && setup_docker_compose && setup_docker_syncer \ + ## Clean up and display components version information... && source /opt/utils/script-utils.sh && install__clean && list_installed_packages diff --git a/docker_docker_kit/work/image-syncer/run_jobs.py b/docker_docker_kit/work/image-syncer/run_jobs.py deleted file mode 100644 index 1bd1570a..00000000 --- a/docker_docker_kit/work/image-syncer/run_jobs.py +++ /dev/null @@ -1,60 +0,0 @@ -import argparse -import multiprocessing -import os -import sys -import yaml - -from run_sync import generate_tasks_with_auth, generate_tasks_without_auth, sync_image - - -def get_job_names_from_yaml(file_path): - """Get all job names from GitHub Actions config file""" - images = [] - with open(file_path, 'r') as file: - try: - yaml_content = yaml.safe_load(file) - # GitHub Actions YAML file structure has a 'jobs' key at its root - jobs = yaml_content.get('jobs', {}) - for _, v in jobs.items(): - name = v.get('name') - if name is not None: - images.extend(name.split(',')) - except yaml.YAMLError as exc: - print(f"Error parsing YAML file: {exc}") - return images - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--workflow-file', type=str, default='.github/workflows/build-docker.yml', help='GitHub actions workflow file') - parser.add_argument('--image-namespace', type=str, default=os.environ.get('IMG_NAMESPACE', 'library'), help='namespace of the image') - parser.add_argument('--source-registry', type=str, default=os.environ.get("REGISTRY_URL", 'docker.io'), help='REGISTRY_URL') - parser.add_argument('--target-registry', type=str, default=os.environ.get("DOCKER_MIRROR_REGISTRY", '').split(','), action='extend', nargs='*', help='Target registry URL') - parser.add_argument('--auth-file', type=str, default=os.environ.get("FILE_AUTH", None), help='auth file used for image-sync') - args = parser.parse_args() - - list_tasks = [] - for image in get_job_names_from_yaml(args.workflow_file): - img = '/'.join([args.image_namespace, image]) - print("Docker image sync job name found:", img) - - param = dict( - image=img, tags=None, source_registry=args.source_registry, target_registries=args.target_registry - ) - - if args.auth_file is not None: - cfg = generate_tasks_without_auth(**param) - list_tasks.append((cfg, args.auth_file)) - else: - cfg = generate_tasks_with_auth(**param) - for _, c in enumerate(cfg): - list_tasks.append((c, None)) - - with multiprocessing.Pool() as pool: - res = pool.starmap_async(sync_image, list_tasks) - ret = sum(res.get()) - return ret - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/docker_docker_kit/work/image-syncer/run_sync.py b/docker_docker_kit/work/image-syncer/run_sync.py deleted file mode 100644 index c771a6a3..00000000 --- a/docker_docker_kit/work/image-syncer/run_sync.py +++ /dev/null @@ -1,97 +0,0 @@ -import argparse -import json -import os -import subprocess -import sys -import tempfile - - -def generate_tasks_without_auth(image: str, source_registry: str = None, target_registries: list = None, tags: list = None): - """Generate a config item which will be used by `image-syncer`.""" - if target_registries is None or len(target_registries) == 0: - destinations = 'cn-beijing,cn-hangzhou'.split(',') # ,cn-shanghai,cn-shenzhen,cn-chengdu,cn-hongkong,us-west-1,eu-central-1 - target_registries = ['registry.%s.aliyuncs.com' % i for i in destinations] - - img_src_tag = '%s:%s' % (image, tags) if tags is not None else image - img_src: str = "%s/%s" % (source_registry, img_src_tag) - img_dst = ["%s/%s" % (target_registry, image) for target_registry in target_registries] - images = {img_src: img_dst} - return images - - -def generate_tasks_with_auth(image: str, source_registry: str = None, target_registries: list = None, tags: list = None): - """Generate a config item which will be used by `image-syncer`.""" - uname_mirror = os.environ.get('DOCKER_MIRROR_REGISTRY_USERNAME', None) - passwd_mirror = os.environ.get('DOCKER_MIRROR_REGISTRY_PASSWORD', None) - - if uname_mirror is None or passwd_mirror is None: - raise ValueError('ENV variable required: DOCKER_MIRROR_REGISTRY_USERNAME and DOCKER_MIRROR_REGISTRY_PASSWORD!') - - auth = {"username": uname_mirror, "password": passwd_mirror} - - if target_registries is None or len(target_registries) == 0: - destinations = 'cn-beijing,cn-hangzhou'.split(',') # ,cn-shanghai,cn-shenzhen,cn-chengdu,cn-hongkong,us-west-1,eu-central-1 - target_registries = ['registry.%s.aliyuncs.com' % i for i in destinations] - - for target_registry in target_registries: - img_src_tag = '%s:%s' % (image, tags) if tags is not None else image - img_src: str = "%s/%s" % (source_registry, img_src_tag) - img_dst: str = "%s/%s" % (target_registry, image) - images = {img_src: img_dst} - - c = { 'images': images, 'auth': { target_registry: auth } } - if source_registry is not None: - uname_source = os.environ.get('DOCKER_REGISTRY_USERNAME', None) - passwd_source = os.environ.get('DOCKER_REGISTRY_PASSWORD', None) - if uname_source is None or passwd_source is None: - print('ENV variable required: DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD!') - sys.exit(-2) - c['auth'].update({source_registry: { - "username": uname_source, "password": passwd_source} - }) - yield c - - -def sync_image(cfg: dict = None, file_path_auth: str = None): - """Run the sync task using `image-syncer` with given config item.""" - fp = tempfile.NamedTemporaryFile(mode='wt', encoding='UTF-8', suffix='.json', delete_on_close=True) - json.dump(cfg, fp, ensure_ascii=False, indent=2, sort_keys=True) - fp.flush() - - if 'auth' in cfg: - opts = ['--config=' + fp.name] - else: - if file_path_auth is None: - raise ValueError('file_path_auth required if auth not provided directly!') - opts = ['--auth=' + file_path_auth, '--images=' + fp.name] - - cmd = ['image-syncer', '--proc=16', '--retries=2', ] + opts - # print('Running cmd:', ' '.join(cmd)) - # print('Job payload:', json.dumps(cfg, ensure_ascii=False, indent=None)) - ret = 0 - try: - subprocess.run(cmd, check=True) - except subprocess.CalledProcessError as e: - ret = e.returncode - print(e) - fp.close() - return ret - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('img', type=str, help='Source image, with or without tag') - parser.add_argument('--tags', type=str, action='extend', nargs='*', help='Tags to sync, optional.') - parser.add_argument('--source-registry', type=str, default='docker.io', help='Target image name, with our without tag') - parser.add_argument('--target-registry', type=str, action='extend', nargs='*', help='Target registry URL') - args = parser.parse_args() - - configs = generate_tasks_with_auth(image=args.img, tags=args.tags, source_registry=args.source_registry, target_registries=args.target_registry) - ret = 0 - for _, c in enumerate(configs): - ret += sync_image(cfg=c) - return ret - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/docker_docker_kit/work/script-sync-image.sh b/docker_docker_kit/work/script-sync-image.sh new file mode 100644 index 00000000..37f62f53 --- /dev/null +++ b/docker_docker_kit/work/script-sync-image.sh @@ -0,0 +1,199 @@ +#!/bin/sh + +usage() { + cat <<'EOF' +Usage: script-sync-image.sh [IMAGE] [options] + --workflow-file FILE GitHub Actions workflow YAML + --image-namespace NAME Image namespace (default: IMG_NAMESPACE or library) + --source-registry NAME Source registry (default: REGISTRY_URL or docker.io) + --target-registry NAME Target registry; repeat or use comma-separated values + --auth-file FILE image-syncer auth file + --image IMAGE Sync one fully-qualified image path instead of workflow jobs + --help Show this help +EOF +} + +workflow_file=.github/workflows/build-docker.yml +image_namespace=${IMG_NAMESPACE:-library} +source_registry=${REGISTRY_URL-docker.io} +target_registries=${DOCKER_MIRROR_REGISTRY:-} +auth_file=${FILE_AUTH:-} +single_image= + +append_targets() { + if [ -n "$target_registries" ]; then + target_registries="${target_registries},$1" + else + target_registries=$1 + fi +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --workflow-file) + [ "$#" -ge 2 ] || { echo "Missing value for --workflow-file" >&2; exit 2; } + workflow_file=$2 + shift 2 + ;; + --workflow-file=*) + workflow_file=${1#*=} + shift + ;; + --image-namespace) + [ "$#" -ge 2 ] || { echo "Missing value for --image-namespace" >&2; exit 2; } + image_namespace=$2 + shift 2 + ;; + --image-namespace=*) + image_namespace=${1#*=} + shift + ;; + --source-registry) + [ "$#" -ge 2 ] || { echo "Missing value for --source-registry" >&2; exit 2; } + source_registry=$2 + shift 2 + ;; + --source-registry=*) + source_registry=${1#*=} + shift + ;; + --target-registry) + shift + found_target=false + while [ "$#" -gt 0 ] && [ "${1#--}" = "$1" ]; do + append_targets "$1" + found_target=true + shift + done + [ "$found_target" = true ] || { echo "Missing value for --target-registry" >&2; exit 2; } + ;; + --target-registry=*) + append_targets "${1#*=}" + shift + ;; + --auth-file) + [ "$#" -ge 2 ] || { echo "Missing value for --auth-file" >&2; exit 2; } + auth_file=$2 + shift 2 + ;; + --auth-file=*) + auth_file=${1#*=} + shift + ;; + --image) + [ "$#" -ge 2 ] || { echo "Missing value for --image" >&2; exit 2; } + single_image=$2 + shift 2 + ;; + --image=*) + single_image=${1#*=} + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + if [ -z "$single_image" ] && [ "${1#-}" = "$1" ]; then + single_image=$1 + shift + else + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + fi + ;; + esac +done + +export SOURCE_REGISTRY=$source_registry + +command -v yq >/dev/null 2>&1 || { echo "Required command not found: yq" >&2; exit 1; } +command -v image-syncer >/dev/null 2>&1 || { echo "Required command not found: image-syncer" >&2; exit 1; } + +tmp_dir=$(mktemp -d) || { echo "Unable to create temporary directory" >&2; exit 1; } +trap 'rm -rf "$tmp_dir"' 0 +trap 'exit 1' HUP INT TERM +names_file=$tmp_dir/names +raw_names_file=$tmp_dir/raw-names +targets_file=$tmp_dir/targets +config_file=$tmp_dir/config.json + +if [ -n "$single_image" ]; then + printf '%s\n' "$single_image" > "$names_file" +else + if [ ! -r "$workflow_file" ]; then + echo "Workflow file is not readable: $workflow_file" >&2 + exit 1 + fi + if ! yq -r '.jobs[]? | select(.name != null) | .name' "$workflow_file" > "$raw_names_file"; then + echo "Failed to parse workflow file: $workflow_file" >&2 + exit 1 + fi + while IFS= read -r job_name || [ -n "$job_name" ]; do + printf '%s\n' "$job_name" | tr ',' '\n' >> "$names_file" + done < "$raw_names_file" +fi + +printf '%s\n' "$target_registries" | tr ',' '\n' | + sed 's/^[[:space:]]*//; s/[[:space:]]*$//; /^$/d' > "$targets_file" +if [ ! -s "$targets_file" ]; then + printf '%s\n' registry.cn-beijing.aliyuncs.com registry.cn-hangzhou.aliyuncs.com > "$targets_file" +fi + +ret=0 +while IFS= read -r image || [ -n "$image" ]; do + if [ -z "$single_image" ]; then + image_path=$image_namespace/$image + else + image_path=$image + fi + source_image=$source_registry/$image_path + echo "Docker image sync job name found: $image_path" + + if [ -n "$auth_file" ]; then + export SOURCE_IMAGE=$source_image TARGET_REGISTRIES= + while IFS= read -r target_registry || [ -n "$target_registry" ]; do + destination=$target_registry/$image_path + if [ -n "$TARGET_REGISTRIES" ]; then + TARGET_REGISTRIES=$TARGET_REGISTRIES,$destination + else + TARGET_REGISTRIES=$destination + fi + done < "$targets_file" + export TARGET_REGISTRIES + if ! yq -n -o=json '{"images": {(strenv(SOURCE_IMAGE)): (strenv(TARGET_REGISTRIES) | split(","))}}' > "$config_file"; then + echo "Failed to generate image config for $image_path" >&2 + ret=$((ret + 1)) + continue + fi + image-syncer --proc=16 --retries=2 "--auth=$auth_file" "--images=$config_file" + status=$? + ret=$((ret + status)) + else + if [ -z "${DOCKER_MIRROR_REGISTRY_USERNAME:-}" ] || [ -z "${DOCKER_MIRROR_REGISTRY_PASSWORD:-}" ]; then + echo "DOCKER_MIRROR_REGISTRY_USERNAME and DOCKER_MIRROR_REGISTRY_PASSWORD are required without --auth-file" >&2 + exit 1 + fi + if [ -z "${DOCKER_REGISTRY_USERNAME:-}" ] || [ -z "${DOCKER_REGISTRY_PASSWORD:-}" ]; then + echo "DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD are required without --auth-file" >&2 + exit 254 + fi + while IFS= read -r target_registry || [ -n "$target_registry" ]; do + destination=$target_registry/$image_path + export SOURCE_IMAGE=$source_image TARGET_REGISTRY=$target_registry DESTINATION=$destination + if ! yq -n -o=json \ + '{"images": {(strenv(SOURCE_IMAGE)): [strenv(DESTINATION)]}, "auth": {(strenv(TARGET_REGISTRY)): {"username": strenv(DOCKER_MIRROR_REGISTRY_USERNAME), "password": strenv(DOCKER_MIRROR_REGISTRY_PASSWORD)}}} * {"auth": {(strenv(SOURCE_REGISTRY)): {"username": strenv(DOCKER_REGISTRY_USERNAME), "password": strenv(DOCKER_REGISTRY_PASSWORD)}}}' \ + > "$config_file"; then + echo "Failed to generate image config for $image_path" >&2 + ret=$((ret + 1)) + continue + fi + image-syncer --proc=16 --retries=2 "--config=$config_file" + status=$? + ret=$((ret + status)) + done < "$targets_file" + fi +done < "$names_file" + +exit "$ret" From 24a317618d4be5cadb2ddc8df0178ac649f4b4ad Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Sat, 14 Nov 2026 20:17:12 +0000 Subject: [PATCH 2/8] update scripts --- docker_docker_kit/docker-kit.Dockerfile | 2 +- docker_docker_kit/work/script-sync-image.sh | 46 ++++++++++----------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docker_docker_kit/docker-kit.Dockerfile b/docker_docker_kit/docker-kit.Dockerfile index db9754e5..ea6bd878 100644 --- a/docker_docker_kit/docker-kit.Dockerfile +++ b/docker_docker_kit/docker-kit.Dockerfile @@ -10,7 +10,7 @@ ENV XDG_RUNTIME_DIR="/tmp" COPY work /opt/utils/ RUN set -eux \ - && chmod +x /opt/utils/*.sh \ + && chmod +x /opt/utils/*.sh && ln -sf /opt/utils/script-sync-image.sh /usr/local/bin/ \ && source /opt/utils/script-setup-core.sh && setup_yq \ && source /opt/utils/script-setup-docker.sh && setup_docker_compose && setup_docker_syncer \ ## Clean up and display components version information... diff --git a/docker_docker_kit/work/script-sync-image.sh b/docker_docker_kit/work/script-sync-image.sh index 37f62f53..9f6fc14f 100644 --- a/docker_docker_kit/work/script-sync-image.sh +++ b/docker_docker_kit/work/script-sync-image.sh @@ -3,21 +3,21 @@ usage() { cat <<'EOF' Usage: script-sync-image.sh [IMAGE] [options] - --workflow-file FILE GitHub Actions workflow YAML + --file-auth FILE image-syncer auth file + --file-workflow FILE GitHub Actions workflow YAML --image-namespace NAME Image namespace (default: IMG_NAMESPACE or library) --source-registry NAME Source registry (default: REGISTRY_URL or docker.io) --target-registry NAME Target registry; repeat or use comma-separated values - --auth-file FILE image-syncer auth file --image IMAGE Sync one fully-qualified image path instead of workflow jobs --help Show this help EOF } -workflow_file=.github/workflows/build-docker.yml +file_workflow=.github/workflows/build-docker.yml +file_auth=${FILE_AUTH:-} image_namespace=${IMG_NAMESPACE:-library} -source_registry=${REGISTRY_URL-docker.io} +source_registry=${REGISTRY_URL:-docker.io} target_registries=${DOCKER_MIRROR_REGISTRY:-} -auth_file=${FILE_AUTH:-} single_image= append_targets() { @@ -30,13 +30,13 @@ append_targets() { while [ "$#" -gt 0 ]; do case "$1" in - --workflow-file) - [ "$#" -ge 2 ] || { echo "Missing value for --workflow-file" >&2; exit 2; } - workflow_file=$2 + --file-workflow) + [ "$#" -ge 2 ] || { echo "Missing value for --file-workflow" >&2; exit 2; } + file_workflow=$2 shift 2 ;; - --workflow-file=*) - workflow_file=${1#*=} + --file-workflow=*) + file_workflow=${1#*=} shift ;; --image-namespace) @@ -71,13 +71,13 @@ while [ "$#" -gt 0 ]; do append_targets "${1#*=}" shift ;; - --auth-file) - [ "$#" -ge 2 ] || { echo "Missing value for --auth-file" >&2; exit 2; } - auth_file=$2 + --file-auth) + [ "$#" -ge 2 ] || { echo "Missing value for --file-auth" >&2; exit 2; } + file_auth=$2 shift 2 ;; - --auth-file=*) - auth_file=${1#*=} + --file-auth=*) + file_auth=${1#*=} shift ;; --image) @@ -122,12 +122,12 @@ config_file=$tmp_dir/config.json if [ -n "$single_image" ]; then printf '%s\n' "$single_image" > "$names_file" else - if [ ! -r "$workflow_file" ]; then - echo "Workflow file is not readable: $workflow_file" >&2 + if [ ! -r "$file_workflow" ]; then + echo "Workflow file is not readable: $file_workflow" >&2 exit 1 fi - if ! yq -r '.jobs[]? | select(.name != null) | .name' "$workflow_file" > "$raw_names_file"; then - echo "Failed to parse workflow file: $workflow_file" >&2 + if ! yq -r '.jobs[]? | select(.name != null) | .name' "$file_workflow" > "$raw_names_file"; then + echo "Failed to parse workflow file: $file_workflow" >&2 exit 1 fi while IFS= read -r job_name || [ -n "$job_name" ]; do @@ -151,7 +151,7 @@ while IFS= read -r image || [ -n "$image" ]; do source_image=$source_registry/$image_path echo "Docker image sync job name found: $image_path" - if [ -n "$auth_file" ]; then + if [ -n "$file_auth" ]; then export SOURCE_IMAGE=$source_image TARGET_REGISTRIES= while IFS= read -r target_registry || [ -n "$target_registry" ]; do destination=$target_registry/$image_path @@ -167,16 +167,16 @@ while IFS= read -r image || [ -n "$image" ]; do ret=$((ret + 1)) continue fi - image-syncer --proc=16 --retries=2 "--auth=$auth_file" "--images=$config_file" + image-syncer --proc=16 --retries=2 "--auth=$file_auth" "--images=$config_file" status=$? ret=$((ret + status)) else if [ -z "${DOCKER_MIRROR_REGISTRY_USERNAME:-}" ] || [ -z "${DOCKER_MIRROR_REGISTRY_PASSWORD:-}" ]; then - echo "DOCKER_MIRROR_REGISTRY_USERNAME and DOCKER_MIRROR_REGISTRY_PASSWORD are required without --auth-file" >&2 + echo "DOCKER_MIRROR_REGISTRY_USERNAME and DOCKER_MIRROR_REGISTRY_PASSWORD are required without --file-auth" >&2 exit 1 fi if [ -z "${DOCKER_REGISTRY_USERNAME:-}" ] || [ -z "${DOCKER_REGISTRY_PASSWORD:-}" ]; then - echo "DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD are required without --auth-file" >&2 + echo "DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD are required without --file-auth" >&2 exit 254 fi while IFS= read -r target_registry || [ -n "$target_registry" ]; do From 95e2f758b885631f596fcf83aed27f7070c30e06 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Sat, 14 Nov 2026 20:33:46 +0000 Subject: [PATCH 3/8] updates --- .github/workflows/build-docker-db.yml | 4 +- .github/workflows/build-docker-gpu.yml | 4 +- .github/workflows/build-docker.yml | 4 +- docker_docker_kit/README.md | 1 + docker_docker_kit/work/script-sync-image.sh | 187 +++++++++++--------- tool.sh | 9 +- 6 files changed, 115 insertions(+), 94 deletions(-) diff --git a/.github/workflows/build-docker-db.yml b/.github/workflows/build-docker-db.yml index e706d75f..b76e285a 100644 --- a/.github/workflows/build-docker-db.yml +++ b/.github/workflows/build-docker-db.yml @@ -61,5 +61,5 @@ jobs: printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \ - sh /opt/utils/script-sync-image.sh --auth-file=/tmp/.github/workflows/auth.json \ - --workflow-file=".github/workflows/build-docker-db.yml" + sh /opt/utils/script-sync-image.sh --file-auth=/tmp/.github/workflows/auth.json \ + --file-workflow=".github/workflows/build-docker-db.yml" diff --git a/.github/workflows/build-docker-gpu.yml b/.github/workflows/build-docker-gpu.yml index 1bda79d0..d39c7961 100644 --- a/.github/workflows/build-docker-gpu.yml +++ b/.github/workflows/build-docker-gpu.yml @@ -166,5 +166,5 @@ jobs: printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \ - sh /opt/utils/script-sync-image.sh --auth-file=/tmp/.github/workflows/auth.json \ - --workflow-file=".github/workflows/build-docker-gpu.yml" + sh /opt/utils/script-sync-image.sh --file-auth=/tmp/.github/workflows/auth.json \ + --file-workflow=".github/workflows/build-docker-gpu.yml" diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 7739c983..e572afe1 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -234,7 +234,7 @@ jobs: printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \ - sh /opt/utils/script-sync-image.sh --auth-file=/tmp/.github/workflows/auth.json + script-sync-image.sh --file-auth=/tmp/.github/workflows/auth.json ## Sync all images in this build (listed by "names") to mirror registry. job-sync-images: @@ -250,4 +250,4 @@ jobs: printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \ - sh /opt/utils/script-sync-image.sh --auth-file=/tmp/.github/workflows/auth.json + script-sync-image.sh --file-auth=/tmp/.github/workflows/auth.json diff --git a/docker_docker_kit/README.md b/docker_docker_kit/README.md index 450cd3f3..531c354c 100644 --- a/docker_docker_kit/README.md +++ b/docker_docker_kit/README.md @@ -24,6 +24,7 @@ docker run --rm \ ``` Fill the environment variables above, and more details can be found in `./work/script-sync-image.sh`. +For workflow-based syncs, `IMG_NAMESPACE_SRC` selects the source namespace and `IMG_NAMESPACE_DST` optionally selects a different destination namespace; when omitted, the destination namespace matches the source. ### Sync images in batch diff --git a/docker_docker_kit/work/script-sync-image.sh b/docker_docker_kit/work/script-sync-image.sh index 9f6fc14f..e2584174 100644 --- a/docker_docker_kit/work/script-sync-image.sh +++ b/docker_docker_kit/work/script-sync-image.sh @@ -3,90 +3,101 @@ usage() { cat <<'EOF' Usage: script-sync-image.sh [IMAGE] [options] - --file-auth FILE image-syncer auth file - --file-workflow FILE GitHub Actions workflow YAML - --image-namespace NAME Image namespace (default: IMG_NAMESPACE or library) - --source-registry NAME Source registry (default: REGISTRY_URL or docker.io) - --target-registry NAME Target registry; repeat or use comma-separated values - --image IMAGE Sync one fully-qualified image path instead of workflow jobs - --help Show this help + --file-auth FILE image-syncer auth file + --file-workflow FILE GitHub Actions workflow YAML + --image-namespace-src NAME Source image namespace (default: IMG_NAMESPACE_SRC or library) + --image-namespace-dst NAME Destination image namespace (default: source namespace) + --source-registry NAME Source registry (default: REGISTRY_URL or docker.io) + --target-registry NAME Target registry; repeat or use comma-separated values + --image IMAGE Sync one fully-qualified image path instead of workflow jobs + --help Show this help EOF } file_workflow=.github/workflows/build-docker.yml file_auth=${FILE_AUTH:-} -image_namespace=${IMG_NAMESPACE:-library} -source_registry=${REGISTRY_URL:-docker.io} -target_registries=${DOCKER_MIRROR_REGISTRY:-} -single_image= +namespace_img_src=${IMG_NAMESPACE_SRC:-library} +namespace_img_dst=${IMG_NAMESPACE_DST:-} +registry_source=${REGISTRY_URL:-docker.io} +registries_target=${DOCKER_MIRROR_REGISTRY:-} +image_single= -append_targets() { - if [ -n "$target_registries" ]; then - target_registries="${target_registries},$1" +append_target_registries() { + if [ -n "$registries_target" ]; then + registries_target="${registries_target},$1" else - target_registries=$1 + registries_target=$1 fi } while [ "$#" -gt 0 ]; do case "$1" in - --file-workflow) - [ "$#" -ge 2 ] || { echo "Missing value for --file-workflow" >&2; exit 2; } + --file-workflow|--workflow-file) + [ "$#" -ge 2 ] || { echo "Missing value for $1" >&2; exit 2; } file_workflow=$2 shift 2 ;; - --file-workflow=*) + --file-workflow=*|--workflow-file=*) file_workflow=${1#*=} shift ;; - --image-namespace) - [ "$#" -ge 2 ] || { echo "Missing value for --image-namespace" >&2; exit 2; } - image_namespace=$2 + --image-namespace-src|--image-namespace) + [ "$#" -ge 2 ] || { echo "Missing value for $1" >&2; exit 2; } + namespace_img_src=$2 shift 2 ;; - --image-namespace=*) - image_namespace=${1#*=} + --image-namespace-src=*|--image-namespace=*) + namespace_img_src=${1#*=} + shift + ;; + --image-namespace-dst) + [ "$#" -ge 2 ] || { echo "Missing value for --image-namespace-dst" >&2; exit 2; } + namespace_img_dst=$2 + shift 2 + ;; + --image-namespace-dst=*) + namespace_img_dst=${1#*=} shift ;; --source-registry) - [ "$#" -ge 2 ] || { echo "Missing value for --source-registry" >&2; exit 2; } - source_registry=$2 + [ "$#" -ge 2 ] || { echo "Missing value for $1" >&2; exit 2; } + registry_source=$2 shift 2 ;; --source-registry=*) - source_registry=${1#*=} + registry_source=${1#*=} shift ;; --target-registry) shift - found_target=false + target_found=false while [ "$#" -gt 0 ] && [ "${1#--}" = "$1" ]; do - append_targets "$1" - found_target=true + append_target_registries "$1" + target_found=true shift done - [ "$found_target" = true ] || { echo "Missing value for --target-registry" >&2; exit 2; } + [ "$target_found" = true ] || { echo "Missing value for --target-registry" >&2; exit 2; } ;; --target-registry=*) - append_targets "${1#*=}" + append_target_registries "${1#*=}" shift ;; - --file-auth) - [ "$#" -ge 2 ] || { echo "Missing value for --file-auth" >&2; exit 2; } + --file-auth|--auth-file) + [ "$#" -ge 2 ] || { echo "Missing value for $1" >&2; exit 2; } file_auth=$2 shift 2 ;; - --file-auth=*) + --file-auth=*|--auth-file=*) file_auth=${1#*=} shift ;; --image) - [ "$#" -ge 2 ] || { echo "Missing value for --image" >&2; exit 2; } - single_image=$2 + [ "$#" -ge 2 ] || { echo "Missing value for $1" >&2; exit 2; } + image_single=$2 shift 2 ;; --image=*) - single_image=${1#*=} + image_single=${1#*=} shift ;; --help|-h) @@ -94,8 +105,8 @@ while [ "$#" -gt 0 ]; do exit 0 ;; *) - if [ -z "$single_image" ] && [ "${1#-}" = "$1" ]; then - single_image=$1 + if [ -z "$image_single" ] && [ "${1#-}" = "$1" ]; then + image_single=$1 shift else echo "Unknown argument: $1" >&2 @@ -106,7 +117,11 @@ while [ "$#" -gt 0 ]; do esac done -export SOURCE_REGISTRY=$source_registry +if [ -z "$namespace_img_dst" ]; then + namespace_img_dst=$namespace_img_src +fi + +export SOURCE_REGISTRY=$registry_source command -v yq >/dev/null 2>&1 || { echo "Required command not found: yq" >&2; exit 1; } command -v image-syncer >/dev/null 2>&1 || { echo "Required command not found: image-syncer" >&2; exit 1; } @@ -114,62 +129,64 @@ command -v image-syncer >/dev/null 2>&1 || { echo "Required command not found: i tmp_dir=$(mktemp -d) || { echo "Unable to create temporary directory" >&2; exit 1; } trap 'rm -rf "$tmp_dir"' 0 trap 'exit 1' HUP INT TERM -names_file=$tmp_dir/names -raw_names_file=$tmp_dir/raw-names -targets_file=$tmp_dir/targets -config_file=$tmp_dir/config.json +file_names=$tmp_dir/names +file_names_raw=$tmp_dir/raw-names +file_targets=$tmp_dir/targets +file_config=$tmp_dir/config.json -if [ -n "$single_image" ]; then - printf '%s\n' "$single_image" > "$names_file" +if [ -n "$image_single" ]; then + printf '%s\n' "$image_single" > "$file_names" else if [ ! -r "$file_workflow" ]; then echo "Workflow file is not readable: $file_workflow" >&2 exit 1 fi - if ! yq -r '.jobs[]? | select(.name != null) | .name' "$file_workflow" > "$raw_names_file"; then + if ! yq -r '.jobs[]? | select(.name != null) | .name' "$file_workflow" > "$file_names_raw"; then echo "Failed to parse workflow file: $file_workflow" >&2 exit 1 fi - while IFS= read -r job_name || [ -n "$job_name" ]; do - printf '%s\n' "$job_name" | tr ',' '\n' >> "$names_file" - done < "$raw_names_file" + while IFS= read -r name_job || [ -n "$name_job" ]; do + printf '%s\n' "$name_job" | tr ',' '\n' >> "$file_names" + done < "$file_names_raw" fi -printf '%s\n' "$target_registries" | tr ',' '\n' | - sed 's/^[[:space:]]*//; s/[[:space:]]*$//; /^$/d' > "$targets_file" -if [ ! -s "$targets_file" ]; then - printf '%s\n' registry.cn-beijing.aliyuncs.com registry.cn-hangzhou.aliyuncs.com > "$targets_file" +printf '%s\n' "$registries_target" | tr ',' '\n' | + sed 's/^[[:space:]]*//; s/[[:space:]]*$//; /^$/d' > "$file_targets" +if [ ! -s "$file_targets" ]; then + printf '%s\n' registry.cn-beijing.aliyuncs.com registry.cn-hangzhou.aliyuncs.com > "$file_targets" fi -ret=0 -while IFS= read -r image || [ -n "$image" ]; do - if [ -z "$single_image" ]; then - image_path=$image_namespace/$image +status_total=0 +while IFS= read -r image_name || [ -n "$image_name" ]; do + if [ -z "$image_single" ]; then + path_image_src=$namespace_img_src/$image_name + path_image_dst=$namespace_img_dst/$image_name else - image_path=$image + path_image_src=$image_name + path_image_dst=$image_name fi - source_image=$source_registry/$image_path - echo "Docker image sync job name found: $image_path" + image_source=$registry_source/$path_image_src + echo "Docker image sync job name found: $path_image_src" if [ -n "$file_auth" ]; then - export SOURCE_IMAGE=$source_image TARGET_REGISTRIES= - while IFS= read -r target_registry || [ -n "$target_registry" ]; do - destination=$target_registry/$image_path + export SOURCE_IMAGE=$image_source TARGET_REGISTRIES= + while IFS= read -r registry_target || [ -n "$registry_target" ]; do + image_destination=$registry_target/$path_image_dst if [ -n "$TARGET_REGISTRIES" ]; then - TARGET_REGISTRIES=$TARGET_REGISTRIES,$destination + TARGET_REGISTRIES=$TARGET_REGISTRIES,$image_destination else - TARGET_REGISTRIES=$destination + TARGET_REGISTRIES=$image_destination fi - done < "$targets_file" + done < "$file_targets" export TARGET_REGISTRIES - if ! yq -n -o=json '{"images": {(strenv(SOURCE_IMAGE)): (strenv(TARGET_REGISTRIES) | split(","))}}' > "$config_file"; then - echo "Failed to generate image config for $image_path" >&2 - ret=$((ret + 1)) + if ! yq -n -o=json '{"images": {(strenv(SOURCE_IMAGE)): (strenv(TARGET_REGISTRIES) | split(","))}}' > "$file_config"; then + echo "Failed to generate image config for $path_image_src" >&2 + status_total=$((status_total + 1)) continue fi - image-syncer --proc=16 --retries=2 "--auth=$file_auth" "--images=$config_file" - status=$? - ret=$((ret + status)) + image-syncer --proc=16 --retries=2 "--auth=$file_auth" "--images=$file_config" + status_sync=$? + status_total=$((status_total + status_sync)) else if [ -z "${DOCKER_MIRROR_REGISTRY_USERNAME:-}" ] || [ -z "${DOCKER_MIRROR_REGISTRY_PASSWORD:-}" ]; then echo "DOCKER_MIRROR_REGISTRY_USERNAME and DOCKER_MIRROR_REGISTRY_PASSWORD are required without --file-auth" >&2 @@ -179,21 +196,21 @@ while IFS= read -r image || [ -n "$image" ]; do echo "DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD are required without --file-auth" >&2 exit 254 fi - while IFS= read -r target_registry || [ -n "$target_registry" ]; do - destination=$target_registry/$image_path - export SOURCE_IMAGE=$source_image TARGET_REGISTRY=$target_registry DESTINATION=$destination + while IFS= read -r registry_target || [ -n "$registry_target" ]; do + image_destination=$registry_target/$path_image_dst + export SOURCE_IMAGE=$image_source TARGET_REGISTRY=$registry_target DESTINATION=$image_destination if ! yq -n -o=json \ '{"images": {(strenv(SOURCE_IMAGE)): [strenv(DESTINATION)]}, "auth": {(strenv(TARGET_REGISTRY)): {"username": strenv(DOCKER_MIRROR_REGISTRY_USERNAME), "password": strenv(DOCKER_MIRROR_REGISTRY_PASSWORD)}}} * {"auth": {(strenv(SOURCE_REGISTRY)): {"username": strenv(DOCKER_REGISTRY_USERNAME), "password": strenv(DOCKER_REGISTRY_PASSWORD)}}}' \ - > "$config_file"; then - echo "Failed to generate image config for $image_path" >&2 - ret=$((ret + 1)) + > "$file_config"; then + echo "Failed to generate image config for $path_image_src" >&2 + status_total=$((status_total + 1)) continue fi - image-syncer --proc=16 --retries=2 "--config=$config_file" - status=$? - ret=$((ret + status)) - done < "$targets_file" + image-syncer --proc=16 --retries=2 "--config=$file_config" + status_sync=$? + status_total=$((status_total + status_sync)) + done < "$file_targets" fi -done < "$names_file" +done < "$file_names" -exit "$ret" +exit "$status_total" diff --git a/tool.sh b/tool.sh index 2c0636ec..f76e39eb 100755 --- a/tool.sh +++ b/tool.sh @@ -21,12 +21,15 @@ CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/') export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})${NAMESPACE_SUFFIX}" ; export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') -export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"quay.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') -export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_NAMESPACE_SRC=$(echo "${IMG_NAMESPACE_SRC:-$IMG_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_NAMESPACE_DST=$(echo "${IMG_NAMESPACE_DST:-$IMG_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"quay.io"}/${IMG_NAMESPACE_SRC}" | awk '{print tolower($0)}') +export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${IMG_NAMESPACE_SRC}" | awk '{print tolower($0)}') export TAG_SUFFIX="-$(git rev-parse --short HEAD)" echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" -echo "--------> DOCKER_IMG_NAMESPACE=${IMG_NAMESPACE}" +echo "--------> DOCKER_IMG_NAMESPACE_SRC=${IMG_NAMESPACE_SRC}" +echo "--------> DOCKER_IMG_NAMESPACE_DST=${IMG_NAMESPACE_DST}" echo "--------> DOCKER_IMG_PREFIX_SRC=${IMG_PREFIX_SRC}" echo "--------> DOCKER_IMG_PREFIX_DST=${IMG_PREFIX_DST}" echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}" From 0de91a4a5750f68b171c6742b9534cd2dcf9c65a Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Sat, 14 Nov 2026 20:48:48 +0000 Subject: [PATCH 4/8] script clean --- .../localize/run-config-mirror-aliyun-pub.sh | 33 +++--- .../localize/run-config-mirror-aliyun-vpc.sh | 33 +++--- .../work/localize/run-config-mirror-cernet.sh | 29 ++--- .../localize/run-config-mirror-default.sh | 12 +- .../localize/run-config-mirror-tencent-pub.sh | 33 +++--- .../localize/run-config-mirror-tencent-vpc.sh | 33 +++--- docker_atom/work/script-setup-core.sh | 103 +++++++++-------- docker_atom/work/script-setup-db-clients.sh | 7 +- docker_atom/work/script-setup-net.sh | 21 ++-- docker_atom/work/script-setup-sys.sh | 22 ++-- docker_atom/work/script-utils.sh | 36 +++--- docker_core/work/script-setup-R.sh | 6 +- .../opt/utils/script-setup-pg-ext-mirror.sh | 5 +- .../rootfs/opt/utils/script-setup-pg-ext.sh | 44 +++---- docker_docker_kit/work/script-setup-docker.sh | 12 +- docker_docker_kit/work/script-sync-image.sh | 108 ++++++------------ tool.sh | 19 +-- 17 files changed, 264 insertions(+), 292 deletions(-) diff --git a/docker_atom/work/localize/run-config-mirror-aliyun-pub.sh b/docker_atom/work/localize/run-config-mirror-aliyun-pub.sh index 21112cc8..3c4e2602 100644 --- a/docker_atom/work/localize/run-config-mirror-aliyun-pub.sh +++ b/docker_atom/work/localize/run-config-mirror-aliyun-pub.sh @@ -3,26 +3,27 @@ set -ex echo "Setup mirror config for Aliyun Public Cloud..." -export TZ=${TZ:="Asia/Shanghai"} -ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone +export TZ="${TZ:-Asia/Shanghai}" +sudo ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" | sudo tee /etc/timezone >/dev/null echo "Setup timezone, current date: $(date)" -eval "export $(cat /etc/os-release | grep ID=)" && export OS_ID=${ID} && echo "Found ${ID} system, setting mirror for ${ID}" +OS_ID=$(sed -n 's/^ID=//p' /etc/os-release | tr -d '"') +echo "Found ${OS_ID} system, setting mirror for ${OS_ID}" -FILE_DEB=$([ -f /etc/apt/sources.list.d/${OS_ID}.sources ] && echo /etc/apt/sources.list.d/${OS_ID}.sources || echo /etc/apt/sources.list ) -if [ -f $FILE_DEB ]; then - sed -i 's|mirrors.*.com/ubuntu|mirrors.aliyun.com/ubuntu|' $FILE_DEB - sed -i 's|archive.ubuntu.com/ubuntu|mirrors.aliyun.com/ubuntu|' $FILE_DEB - sed -i 's|security.ubuntu.com/ubuntu|mirrors.aliyun.com/ubuntu|' $FILE_DEB - sed -i 's|deb.debian.org/debian|mirrors.aliyun.com/debian|' $FILE_DEB +FILE_DEB=$([ -f "/etc/apt/sources.list.d/${OS_ID}.sources" ] && echo "/etc/apt/sources.list.d/${OS_ID}.sources" || echo /etc/apt/sources.list) +if [ -f "$FILE_DEB" ]; then + sudo sed -i 's|mirrors.*.com/ubuntu|mirrors.aliyun.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|archive.ubuntu.com/ubuntu|mirrors.aliyun.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|security.ubuntu.com/ubuntu|mirrors.aliyun.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|deb.debian.org/debian|mirrors.aliyun.com/debian|' "$FILE_DEB" echo "Finished setting ubuntu/debian mirror" fi -[ -f /etc/apt/sources.list.d/pgdg.list ] && sed -i 's|apt.postgresql.org/pub|mirrors.aliyun.com/postgresql|' /etc/apt/sources.list.d/pgdg.list +[ -f /etc/apt/sources.list.d/pgdg.list ] && sudo sed -i 's|apt.postgresql.org/pub|mirrors.aliyun.com/postgresql|' /etc/apt/sources.list.d/pgdg.list -if [ -f "$(which python)" ] ; then +if command -v python >/dev/null 2>&1 ; then echo "Found python, setting pypi source in /etc/pip.conf" - cat >/etc/pip.conf </dev/null </dev/null 2>&1 ; then echo "Found $cmd, setting mirror" "$cmd" config set registry https://registry.npmmirror.com "$cmd" config list @@ -43,14 +44,14 @@ for cmd in npm pnpm yarn bun; do fi done -if [ -f "$(which go)" ] ; then +if command -v go >/dev/null 2>&1 ; then echo "Found golang, setting GOPROXY" export GOPROXY=https://mirrors.aliyun.com/goproxy/ go env | grep 'PROXY' fi -if [ -f "$(which R)" ] ; then +if command -v R >/dev/null 2>&1 ; then echo "Found R, setting CRAN mirror" - echo "options(repos=structure(c(CRAN=\"https://mirrors.aliyun.com/CRAN/\")))" >> /etc/R/Rprofile.site + echo "options(repos=structure(c(CRAN=\"https://mirrors.aliyun.com/CRAN/\")))" | sudo tee -a /etc/R/Rprofile.site >/dev/null R -e "options('repos');" fi diff --git a/docker_atom/work/localize/run-config-mirror-aliyun-vpc.sh b/docker_atom/work/localize/run-config-mirror-aliyun-vpc.sh index 7a3cc31b..6e61ffc7 100644 --- a/docker_atom/work/localize/run-config-mirror-aliyun-vpc.sh +++ b/docker_atom/work/localize/run-config-mirror-aliyun-vpc.sh @@ -3,26 +3,27 @@ set -ex echo "Setup mirror config for Aliyun Cloud VPC Environment..." -export TZ=${TZ:="Asia/Shanghai"} -ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone +export TZ="${TZ:-Asia/Shanghai}" +sudo ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" | sudo tee /etc/timezone >/dev/null echo "Setup timezone, current date: $(date)" -eval "export $(cat /etc/os-release | grep ID=)" && export OS_ID=${ID} && echo "Found ${ID} system, setting mirror for ${ID}" +OS_ID=$(sed -n 's/^ID=//p' /etc/os-release | tr -d '"') +echo "Found ${OS_ID} system, setting mirror for ${OS_ID}" -FILE_DEB=$([ -f /etc/apt/sources.list.d/${OS_ID}.sources ] && echo /etc/apt/sources.list.d/${OS_ID}.sources || echo /etc/apt/sources.list ) -if [ -f $FILE_DEB ]; then - sed -i 's|mirrors.*.com/ubuntu|mirrors.cloud.aliyuncs.com/ubuntu|' $FILE_DEB - sed -i 's|archive.ubuntu.com/ubuntu|mirrors.cloud.aliyuncs.com/ubuntu|' $FILE_DEB - sed -i 's|security.ubuntu.com/ubuntu|mirrors.cloud.aliyuncs.com/ubuntu|' $FILE_DEB - sed -i 's|deb.debian.org/debian|mirrors.cloud.aliyuncs.com/debian|' $FILE_DEB +FILE_DEB=$([ -f "/etc/apt/sources.list.d/${OS_ID}.sources" ] && echo "/etc/apt/sources.list.d/${OS_ID}.sources" || echo /etc/apt/sources.list) +if [ -f "$FILE_DEB" ]; then + sudo sed -i 's|mirrors.*.com/ubuntu|mirrors.cloud.aliyuncs.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|archive.ubuntu.com/ubuntu|mirrors.cloud.aliyuncs.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|security.ubuntu.com/ubuntu|mirrors.cloud.aliyuncs.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|deb.debian.org/debian|mirrors.cloud.aliyuncs.com/debian|' "$FILE_DEB" echo "Finished setting ubuntu/debian mirror" fi -[ -f /etc/apt/sources.list.d/pgdg.list ] && sed -i 's|apt.postgresql.org/pub|mirrors.cloud.aliyuncs.com/postgresql|' /etc/apt/sources.list.d/pgdg.list +[ -f /etc/apt/sources.list.d/pgdg.list ] && sudo sed -i 's|apt.postgresql.org/pub|mirrors.cloud.aliyuncs.com/postgresql|' /etc/apt/sources.list.d/pgdg.list -if [ -f "$(which python)" ] ; then +if command -v python >/dev/null 2>&1 ; then echo "Found python, setting pypi source in /etc/pip.conf" - cat >/etc/pip.conf </dev/null </dev/null 2>&1 ; then echo "Found $cmd, setting mirror" "$cmd" config set registry https://registry.npmmirror.com "$cmd" config list @@ -43,14 +44,14 @@ for cmd in npm pnpm yarn bun; do fi done -if [ -f "$(which go)" ] ; then +if command -v go >/dev/null 2>&1 ; then echo "Found golang, setting GOPROXY" export GOPROXY=https://mirrors.cloud.aliyuncs.com/goproxy/ go env | grep 'PROXY' fi -if [ -f "$(which R)" ] ; then +if command -v R >/dev/null 2>&1 ; then echo "Found R, setting CRAN mirror" - echo "options(repos=structure(c(CRAN=\"http://mirrors.cloud.aliyuncs.com/CRAN/\")))" >> /etc/R/Rprofile.site + echo "options(repos=structure(c(CRAN=\"http://mirrors.cloud.aliyuncs.com/CRAN/\")))" | sudo tee -a /etc/R/Rprofile.site >/dev/null R -e "options('repos');" fi diff --git a/docker_atom/work/localize/run-config-mirror-cernet.sh b/docker_atom/work/localize/run-config-mirror-cernet.sh index 56f9c100..672d2acc 100755 --- a/docker_atom/work/localize/run-config-mirror-cernet.sh +++ b/docker_atom/work/localize/run-config-mirror-cernet.sh @@ -3,26 +3,27 @@ set -ex echo "Setup mirror config for CERNET..." -export TZ=${TZ:="Asia/Shanghai"} -ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone +export TZ="${TZ:-Asia/Shanghai}" +sudo ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" | sudo tee /etc/timezone >/dev/null echo "Setup timezone, current date: $(date)" -eval "export $(cat /etc/os-release | grep ID=)" && export OS_ID=${ID} && echo "Found ${ID} system, setting mirror for ${ID}" +OS_ID=$(sed -n 's/^ID=//p' /etc/os-release | tr -d '"') +echo "Found ${OS_ID} system, setting mirror for ${OS_ID}" FILE_DEB=$([ -f /etc/apt/sources.list.d/${OS_ID}.sources ] && echo /etc/apt/sources.list.d/${OS_ID}.sources || echo /etc/apt/sources.list) if [ -f "$FILE_DEB" ]; then - sed -i 's|mirrors.*.com/ubuntu|mirrors.cernet.edu.cn/ubuntu|g' $FILE_DEB - sed -i 's|archive.ubuntu.com/ubuntu|mirrors.cernet.edu.cn/ubuntu|g' $FILE_DEB - sed -i 's|security.ubuntu.com/ubuntu|mirrors.cernet.edu.cn/ubuntu|g' $FILE_DEB - sed -i 's|deb.debian.org/debian|mirrors.cernet.edu.cn/debian|g' $FILE_DEB + sudo sed -i 's|mirrors.*.com/ubuntu|mirrors.cernet.edu.cn/ubuntu|g' "$FILE_DEB" + sudo sed -i 's|archive.ubuntu.com/ubuntu|mirrors.cernet.edu.cn/ubuntu|g' "$FILE_DEB" + sudo sed -i 's|security.ubuntu.com/ubuntu|mirrors.cernet.edu.cn/ubuntu|g' "$FILE_DEB" + sudo sed -i 's|deb.debian.org/debian|mirrors.cernet.edu.cn/debian|g' "$FILE_DEB" echo "Finished setting Ubuntu/Debian mirror" fi -[ -f /etc/apt/sources.list.d/pgdg.list ] && sed -i 's|apt.postgresql.org/pub|mirrors.cernet.edu.cn/postgresql|' /etc/apt/sources.list.d/pgdg.list +[ -f /etc/apt/sources.list.d/pgdg.list ] && sudo sed -i 's|apt.postgresql.org/pub|mirrors.cernet.edu.cn/postgresql|' /etc/apt/sources.list.d/pgdg.list -if [ -f "$(which python)" ] ; then +if command -v python >/dev/null 2>&1 ; then echo "Found python, setting pypi source in /etc/pip.conf" - cat >/etc/pip.conf </dev/null </dev/null 2>&1 ; then echo "Found $cmd, setting mirror" "$cmd" config set registry https://registry.npmmirror.com "$cmd" config list @@ -43,14 +44,14 @@ for cmd in npm pnpm yarn bun; do fi done -if [ -f "$(which go)" ] ; then +if command -v go >/dev/null 2>&1 ; then echo "Found golang, setting GOPROXY" export GOPROXY=https://mirrors.aliyun.com/goproxy/ go env | grep 'PROXY' fi -if [ -f "$(which R)" ] ; then +if command -v R >/dev/null 2>&1 ; then echo "Found R, setting CRAN mirror" - echo 'options(repos=structure(c(CRAN="https://mirrors.cernet.edu.cn/CRAN/")))' >> /etc/R/Rprofile.site + echo 'options(repos=structure(c(CRAN="https://mirrors.cernet.edu.cn/CRAN/")))' | sudo tee -a /etc/R/Rprofile.site >/dev/null R -e "options('repos');" fi diff --git a/docker_atom/work/localize/run-config-mirror-default.sh b/docker_atom/work/localize/run-config-mirror-default.sh index 81a05571..1efdd5f9 100644 --- a/docker_atom/work/localize/run-config-mirror-default.sh +++ b/docker_atom/work/localize/run-config-mirror-default.sh @@ -1,30 +1,30 @@ #! /bin/sh set -ex -export TZ=${TZ:="Asia/Shanghai"} -ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone +export TZ="${TZ:-Asia/Shanghai}" +sudo ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" | sudo tee /etc/timezone >/dev/null echo "Setup timezone, current date: $(date)" if [ -f /etc/apt/sources.list ] ; then echo "Found Ubuntu/debian system, using default ubuntu/debian mirror" fi -if [ -f "$(which python)" ] ; then +if command -v python >/dev/null 2>&1 ; then echo "Found python, using default pypi source in /etc/pip.conf" pip config list fi -if [ -f "$(which npm)" ] ; then +if command -v npm >/dev/null 2>&1 ; then echo "Found npm, using default npm mirror" npm config list fi -if [ -f "$(which go)" ] ; then +if command -v go >/dev/null 2>&1 ; then echo "Found golang, getting GO env:" go env | sort fi -if [ -f "$(which R)" ] ; then +if command -v R >/dev/null 2>&1 ; then echo "Found R, getting CRAN mirror" R -e "options('repos');" fi diff --git a/docker_atom/work/localize/run-config-mirror-tencent-pub.sh b/docker_atom/work/localize/run-config-mirror-tencent-pub.sh index 09033540..fb50a7b0 100644 --- a/docker_atom/work/localize/run-config-mirror-tencent-pub.sh +++ b/docker_atom/work/localize/run-config-mirror-tencent-pub.sh @@ -3,26 +3,27 @@ set -ex echo "Setup mirror config for Tencent Cloud VPC Environment..." -export TZ=${TZ:="Asia/Shanghai"} -ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone +export TZ="${TZ:-Asia/Shanghai}" +sudo ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" | sudo tee /etc/timezone >/dev/null echo "Setup timezone, current date: $(date)" -eval "export $(cat /etc/os-release | grep ID=)" && export OS_ID=${ID} && echo "Found ${ID} system, setting mirror for ${ID}" +OS_ID=$(sed -n 's/^ID=//p' /etc/os-release | tr -d '"') +echo "Found ${OS_ID} system, setting mirror for ${OS_ID}" -FILE_DEB=$([ -f /etc/apt/sources.list.d/${OS_ID}.sources ] && echo /etc/apt/sources.list.d/${OS_ID}.sources || echo /etc/apt/sources.list ) -if [ -f $FILE_DEB ]; then - sed -i 's|mirrors.*.com/ubuntu|mirrors.tencent.com/ubuntu|' $FILE_DEB - sed -i 's|archive.ubuntu.com/ubuntu|mirrors.tencent.com/ubuntu|' $FILE_DEB - sed -i 's|security.ubuntu.com/ubuntu|mirrors.tencent.com/ubuntu|' $FILE_DEB - sed -i 's|deb.debian.org/debian|mirrors.tencent.com/debian|' $FILE_DEB +FILE_DEB=$([ -f "/etc/apt/sources.list.d/${OS_ID}.sources" ] && echo "/etc/apt/sources.list.d/${OS_ID}.sources" || echo /etc/apt/sources.list) +if [ -f "$FILE_DEB" ]; then + sudo sed -i 's|mirrors.*.com/ubuntu|mirrors.tencent.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|archive.ubuntu.com/ubuntu|mirrors.tencent.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|security.ubuntu.com/ubuntu|mirrors.tencent.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|deb.debian.org/debian|mirrors.tencent.com/debian|' "$FILE_DEB" echo "Finished setting ubuntu/debian mirror" fi -[ -f /etc/apt/sources.list.d/pgdg.list ] && sed -i 's|apt.postgresql.org/pub|mirrors.tencent.com/postgresql|' /etc/apt/sources.list.d/pgdg.list +[ -f /etc/apt/sources.list.d/pgdg.list ] && sudo sed -i 's|apt.postgresql.org/pub|mirrors.tencent.com/postgresql|' /etc/apt/sources.list.d/pgdg.list -if [ -f "$(which python)" ] ; then +if command -v python >/dev/null 2>&1 ; then echo "Found python, setting pypi source in /etc/pip.conf" - cat >/etc/pip.conf </dev/null </dev/null 2>&1 ; then echo "Found $cmd, setting mirror" "$cmd" config set registry https://registry.npmmirror.com "$cmd" config list @@ -43,14 +44,14 @@ for cmd in npm pnpm yarn bun; do fi done -if [ -f "$(which go)" ] ; then +if command -v go >/dev/null 2>&1 ; then echo "Found golang, setting GOPROXY" export GOPROXY=https://mirrors.tencent.com/go/ go env | grep 'PROXY' fi -if [ -f "$(which R)" ] ; then +if command -v R >/dev/null 2>&1 ; then echo "Found R, setting CRAN mirror" - echo "options(repos=structure(c(CRAN=\"http://mirrors.tencent.com/CRAN/\")))" >> /etc/R/Rprofile.site + echo "options(repos=structure(c(CRAN=\"http://mirrors.tencent.com/CRAN/\")))" | sudo tee -a /etc/R/Rprofile.site >/dev/null R -e "options('repos');" fi diff --git a/docker_atom/work/localize/run-config-mirror-tencent-vpc.sh b/docker_atom/work/localize/run-config-mirror-tencent-vpc.sh index d50f45ea..2329f300 100644 --- a/docker_atom/work/localize/run-config-mirror-tencent-vpc.sh +++ b/docker_atom/work/localize/run-config-mirror-tencent-vpc.sh @@ -3,26 +3,27 @@ set -ex echo "Setup mirror config for Tencent Cloud VPC Environment..." -export TZ=${TZ:="Asia/Shanghai"} -ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone +export TZ="${TZ:-Asia/Shanghai}" +sudo ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" | sudo tee /etc/timezone >/dev/null echo "Setup timezone, current date: $(date)" -eval "export $(cat /etc/os-release | grep ID=)" && export OS_ID=${ID} && echo "Found ${ID} system, setting mirror for ${ID}" +OS_ID=$(sed -n 's/^ID=//p' /etc/os-release | tr -d '"') +echo "Found ${OS_ID} system, setting mirror for ${OS_ID}" -FILE_DEB=$([ -f /etc/apt/sources.list.d/${OS_ID}.sources ] && echo /etc/apt/sources.list.d/${OS_ID}.sources || echo /etc/apt/sources.list ) -if [ -f $FILE_DEB ]; then - sed -i 's|mirrors.*.com/ubuntu|mirrors.tencentyun.com/ubuntu|' $FILE_DEB - sed -i 's|archive.ubuntu.com/ubuntu|mirrors.tencentyun.com/ubuntu|' $FILE_DEB - sed -i 's|security.ubuntu.com/ubuntu|mirrors.tencentyun.com/ubuntu|' $FILE_DEB - sed -i 's|deb.debian.org/debian|mirrors.tencentyun.com/debian|' $FILE_DEB +FILE_DEB=$([ -f "/etc/apt/sources.list.d/${OS_ID}.sources" ] && echo "/etc/apt/sources.list.d/${OS_ID}.sources" || echo /etc/apt/sources.list) +if [ -f "$FILE_DEB" ]; then + sudo sed -i 's|mirrors.*.com/ubuntu|mirrors.tencentyun.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|archive.ubuntu.com/ubuntu|mirrors.tencentyun.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|security.ubuntu.com/ubuntu|mirrors.tencentyun.com/ubuntu|' "$FILE_DEB" + sudo sed -i 's|deb.debian.org/debian|mirrors.tencentyun.com/debian|' "$FILE_DEB" echo "Finished setting ubuntu/debian mirror" fi -[ -f /etc/apt/sources.list.d/pgdg.list ] && sed -i 's|apt.postgresql.org/pub|mirrors.tencentyun.com/postgresql|' /etc/apt/sources.list.d/pgdg.list +[ -f /etc/apt/sources.list.d/pgdg.list ] && sudo sed -i 's|apt.postgresql.org/pub|mirrors.tencentyun.com/postgresql|' /etc/apt/sources.list.d/pgdg.list -if [ -f "$(which python)" ] ; then +if command -v python >/dev/null 2>&1 ; then echo "Found python, setting pypi source in /etc/pip.conf" - cat >/etc/pip.conf </dev/null </dev/null 2>&1 ; then echo "Found $cmd, setting mirror" "$cmd" config set registry https://registry.npmmirror.com "$cmd" config list @@ -43,14 +44,14 @@ for cmd in npm pnpm yarn bun; do fi done -if [ -f "$(which go)" ] ; then +if command -v go >/dev/null 2>&1 ; then echo "Found golang, setting GOPROXY" export GOPROXY=https://mirrors.tencentyun.com/go/ go env | grep 'PROXY' fi -if [ -f "$(which R)" ] ; then +if command -v R >/dev/null 2>&1 ; then echo "Found R, setting CRAN mirror" - echo "options(repos=structure(c(CRAN=\"http://mirrors.tencentyun.com/CRAN/\")))" >> /etc/R/Rprofile.site + echo "options(repos=structure(c(CRAN=\"http://mirrors.tencentyun.com/CRAN/\")))" | sudo tee -a /etc/R/Rprofile.site >/dev/null R -e "options('repos');" fi diff --git a/docker_atom/work/script-setup-core.sh b/docker_atom/work/script-setup-core.sh index 12460ef9..e182c8da 100755 --- a/docker_atom/work/script-setup-core.sh +++ b/docker_atom/work/script-setup-core.sh @@ -17,10 +17,10 @@ setup_mamba() { && [ -n "${VER_MICROMAMBA}" ] \ && local URL_MICROMAMBA="https://github.com/mamba-org/micromamba-releases/releases/download/${VER_MICROMAMBA}/micromamba-${UNAME}-${ARCH}.tar.bz2" \ && echo "Downloading micromamba ${VER_MICROMAMBA} from ${URL_MICROMAMBA}" \ - && mkdir -pv /opt/mamba /etc/conda \ - && install_tar_bz $URL_MICROMAMBA bin/micromamba && mv /opt/bin/micromamba /opt/mamba/mamba \ - && ln -sf /opt/mamba/mamba /usr/bin/ \ - && touch /etc/conda/.condarc && ln -sf /etc/conda/.condarc /opt/conda/.condarc \ + && sudo mkdir -pv /opt/mamba /etc/conda \ + && install_tar_bz "$URL_MICROMAMBA" bin/micromamba && sudo mv /opt/bin/micromamba /opt/mamba/mamba \ + && sudo ln -sf /opt/mamba/mamba /usr/bin/ \ + && sudo touch /etc/conda/.condarc && sudo ln -sf /etc/conda/.condarc /opt/conda/.condarc \ && printf "channels:\n" | sudo tee -a /etc/conda/.condarc \ && printf " - conda-forge\n" | sudo tee -a /etc/conda/.condarc \ && cat /etc/conda/.condarc ; @@ -33,7 +33,7 @@ setup_conda_postprocess() { # If python exists, set pypi source if [ -f "$(which python)" ]; then - cat >/etc/pip.conf </dev/null </dev/null) \ @@ -118,25 +118,25 @@ setup_java_base() { local VER_JDK="$VER_JDK_MAJOR" ; if [ "$VER_JDK_MAJOR" = "latest" ] || { case "$VER_JDK_MAJOR" in ''|*[!0-9]*) false ;; *) [ "$VER_JDK_MAJOR" -gt 20 ] ;; esac; }; then - PAGE_JDK_DOWNLOAD="https://www.oracle.com/java/technologies/downloads/" - PAGE_JDK=$(curl -sL --fail "$PAGE_JDK_DOWNLOAD" || { echo "Failed to fetch Oracle JDK download page" && return 1; } ) + local PAGE_JDK_DOWNLOAD="https://www.oracle.com/java/technologies/downloads/" + local PAGE_JDK=$(curl -sL --fail "$PAGE_JDK_DOWNLOAD" || { echo "Failed to fetch Oracle JDK download page" && return 1; } ) if [ "$VER_JDK_MAJOR" = "latest" ]; then VER_JDK=$(echo "$PAGE_JDK" | grep -oE 'jdk-[0-9]+' | sed 's/jdk-//' | sort -rV | head -1) ; fi ; - URL_JDK_DOWNLOAD=$(echo "${PAGE_JDK}" | grep "tar.gz" | grep "http" | grep -v sha256 | grep ${ARCH} | grep -i $(uname) | grep -oP "(https?://[^\s<>\'\"]*)" | grep "jdk-${VER_JDK}" | head -n 1) \ - && VER_JDK_MINOR=$(echo $URL_JDK_DOWNLOAD | grep -Po '[\d\.]{3,}' | head -n1) ; + local URL_JDK_DOWNLOAD=$(echo "${PAGE_JDK}" | grep "tar.gz" | grep "http" | grep -v sha256 | grep "${ARCH}" | grep -i "$(uname)" | grep -oP "(https?://[^\s<>\'\"]*)" | grep "jdk-${VER_JDK}" | head -n 1) \ + && local VER_JDK_MINOR=$(echo "$URL_JDK_DOWNLOAD" | grep -Po '[\d\.]{3,}' | head -n1) ; else - URL_JDK_adoptium="https://api.github.com/repos/adoptium/temurin${VER_JDK_MAJOR}-binaries/releases/latest" \ - && URL_JDK_DOWNLOAD=$( - curl -sL $URL_JDK_adoptium | grep 'tar.gz' | grep -vE '.sha256|.sig|.json|debug|test' | grep ${ARCH} | grep -i $(uname) \ - | grep -oP "(https?://[^\s<>\'\"]*)" | grep -E $(if [ "$IS_ALPINE" = true ]; then echo 'alpine'; else echo -v 'alpine'; fi) | head -n1 + local URL_JDK_adoptium="https://api.github.com/repos/adoptium/temurin${VER_JDK_MAJOR}-binaries/releases/latest" \ + && local URL_JDK_DOWNLOAD=$( + curl -sL "$URL_JDK_adoptium" | grep 'tar.gz' | grep -vE '.sha256|.sig|.json|debug|test' | grep "${ARCH}" | grep -i "$(uname)" \ + | grep -oP "(https?://[^\s<>\'\"]*)" | grep -E "$(if [ "$IS_ALPINE" = true ]; then echo 'alpine'; else echo -v 'alpine'; fi)" | head -n1 ) ; fi echo "Installing JDK version ${VER_JDK} from: ${URL_JDK_DOWNLOAD}" ; - install_tar_gz "${URL_JDK_DOWNLOAD}" && mv /opt/jdk* /opt/jdk && ln -sf /opt/jdk/bin/* /usr/bin/ ; + install_tar_gz "${URL_JDK_DOWNLOAD}" && sudo mv /opt/jdk* /opt/jdk && sudo ln -sf /opt/jdk/bin/* /usr/bin/ ; type java && echo "@ Version of Java (java): $(java -version)" || return 1 ; type javac && echo "@ Version of Java (javac): $(javac -version)" || return 1 ; } @@ -153,8 +153,8 @@ setup_java_maven() { && local URL_MAVEN="https://archive.apache.org/dist/maven/maven-3/${VER_MAVEN}/binaries/apache-maven-${VER_MAVEN}-bin.zip" \ && echo "Downloading Maven version ${VER_MAVEN} from: ${URL_MAVEN}" \ && install_zip "${URL_MAVEN}" \ - && mv "/opt/apache-maven-${VER_MAVEN}" /opt/maven \ - && ln -sf /opt/maven/bin/mvn* /usr/bin/ ; + && sudo mv "/opt/apache-maven-${VER_MAVEN}" /opt/maven \ + && sudo ln -sf /opt/maven/bin/mvn* /usr/bin/ ; type mvn && echo "@ Version of Maven: $(mvn --version)" || return 1 ; } @@ -174,11 +174,11 @@ setup_node_base() { && local URL_NODEJS="https://nodejs.org/dist/v${VER_NODEJS}/node-v${VER_NODEJS}-${UNAME}-${ARCH}.tar.gz" \ && echo "Downloading NodeJS version ${VER_NODEJS} from: ${URL_NODEJS}" \ && install_tar_gz "${URL_NODEJS}" \ - && mv /opt/node* /opt/node \ - && ln -sf /opt/node/bin/n* /usr/bin/ \ + && sudo mv /opt/node* /opt/node \ + && sudo ln -sf /opt/node/bin/n* /usr/bin/ \ && echo 'export PATH=${PATH}:/opt/node/bin' | sudo tee -a /etc/profile.d/path-node.sh \ && . /etc/profile.d/path-node.sh \ - && npm install -g npm ; + && sudo npm install -g npm ; # cd /tmp && corepack enable && yarn set version stable && echo "@ Version of Yarn: $(yarn -v)" type node && echo "@ Version of Node and node: $(node -v)" || return 1 ; type npm && echo "@ Version of Node and npm: $(npm -v)" || return 1 ; @@ -197,20 +197,20 @@ setup_node_pnpm() { local VER_PNPM=$(echo "${VERS_PNPM}" | head -1) fi \ && [ -n "${VER_PNPM}" ] \ - && mkdir -p /opt/node/bin /opt/node/pnpm-store \ + && sudo mkdir -p /opt/node/bin /opt/node/pnpm-store \ && if [ "${VER_PNPM%%.*}" -ge 11 ]; then URL_PNPM="https://github.com/pnpm/pnpm/releases/download/v${VER_PNPM}/pnpm-${UNAME}-${ARCH}.tar.gz" \ && echo "@ Installing pnpm v${VER_PNPM} from: ${URL_PNPM}" \ && TMPDIR=$(mktemp -d) && curl -fL "${URL_PNPM}" | tar -xzf - -C "${TMPDIR}" \ - && mv -f "${TMPDIR}/pnpm" /opt/node/bin/pnpm \ - && { [ -d "${TMPDIR}/dist" ] && rm -rf /opt/node/bin/dist && mv "${TMPDIR}/dist" /opt/node/bin/ ; rm -rf "${TMPDIR}" ; true ; } ; + && sudo mv -f "${TMPDIR}/pnpm" /opt/node/bin/pnpm \ + && { [ -d "${TMPDIR}/dist" ] && sudo rm -rf /opt/node/bin/dist && sudo mv "${TMPDIR}/dist" /opt/node/bin/ ; rm -rf "${TMPDIR}" ; true ; } ; else URL_PNPM="https://github.com/pnpm/pnpm/releases/download/v${VER_PNPM}/pnpm-${UNAME}-${ARCH}" \ && echo "@ Installing pnpm v${VER_PNPM} from: ${URL_PNPM}" \ - && curl -fL "${URL_PNPM}" -o /opt/node/bin/pnpm ; + && sudo curl -fL "${URL_PNPM}" -o /opt/node/bin/pnpm ; fi \ - && chmod +x /opt/node/bin/pnpm && ln -sf /opt/node/bin/pnpm /usr/local/bin/pnpm \ - && echo 'export PNPM_STORE_DIR=/opt/node/pnpm-store' >/etc/profile.d/path-pnpm.sh ; + && sudo chmod +x /opt/node/bin/pnpm && sudo ln -sf /opt/node/bin/pnpm /usr/local/bin/pnpm \ + && echo 'export PNPM_STORE_DIR=/opt/node/pnpm-store' | sudo tee /etc/profile.d/path-pnpm.sh >/dev/null ; type pnpm >/dev/null 2>&1 && echo "@ Version of pnpm: $(pnpm --version)" || return 1 ; } @@ -282,11 +282,11 @@ setup_rust() { setup_R_base() { local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && curl -sL https://cloud.r-project.org/bin/${UNAME}/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc \ - && echo "deb https://cloud.r-project.org/bin/${UNAME}/ubuntu $(lsb_release -cs)-cran40/" > /etc/apt/sources.list.d/cran.list \ + && echo "deb https://cloud.r-project.org/bin/${UNAME}/ubuntu $(lsb_release -cs)-cran40/" | sudo tee /etc/apt/sources.list.d/cran.list >/dev/null \ && install_apt /opt/utils/install_list_R_base.apt \ && echo "options(repos=structure(c(CRAN=\"https://cloud.r-project.org\")))" | sudo tee -a /etc/R/Rprofile.site \ - && R -e "install.packages(c('devtools'),clean=T,quiet=T);" \ - && R -e "install.packages(c('devtools'),clean=T,quiet=F);" \ + && sudo R -e "install.packages(c('devtools'),clean=T,quiet=T);" \ + && sudo R -e "install.packages(c('devtools'),clean=T,quiet=F);" \ && ( type java && type R && R CMD javareconf || true ) ; type R && echo "@ Version of R: $(R --version)" || return 1 ; } @@ -320,22 +320,21 @@ setup_julia() { setup_lua_base() { - VER_LUA_REQ="${1:-}" \ - && VER_LUA_REQ="${VER_LUA_REQ#v}" \ - && LUA_HOME="${LUA_HOME:-/opt/lua}" \ - && VERS_JSON=$(curl -fsSL "https://api.github.com/repos/lua/lua/releases?per_page=50") \ - && VERS_LUA=$(printf '%s\n' "$VERS_JSON" | sed -n 's/.*"tag_name"[[:space:]]*:[[:space:]]*"v\([0-9][0-9.]*\)".*/\1/p') \ + local VER_LUA_REQ="${1:-}" && VER_LUA_REQ="${VER_LUA_REQ#v}" \ + && local LUA_HOME="${LUA_HOME:-/opt/lua}" \ + && local VERS_JSON=$(curl -fsSL "https://api.github.com/repos/lua/lua/releases?per_page=50") \ + && local VERS_LUA=$(printf '%s\n' "$VERS_JSON" | sed -n 's/.*"tag_name"[[:space:]]*:[[:space:]]*"v\([0-9][0-9.]*\)".*/\1/p') \ && if [ -n "$VER_LUA_REQ" ]; then - VER_LUA=$(printf '%s\n' "$VERS_LUA" | grep -F -x "$VER_LUA_REQ" | sed -n '1p') + local VER_LUA=$(printf '%s\n' "$VERS_LUA" | grep -F -x "$VER_LUA_REQ" | sed -n '1p') [ -n "$VER_LUA" ] || { echo "Lua version not found: $VER_LUA_REQ" >&2; exit 1; } else - VER_LUA=$(printf '%s\n' "$VERS_LUA" | sed -n '1p') + local VER_LUA=$(printf '%s\n' "$VERS_LUA" | sed -n '1p') fi \ && { [ -n "$VER_LUA" ] || { echo "No Lua release found" >&2; exit 1; }; } \ - && URL_LUA="https://www.lua.org/ftp/lua-${VER_LUA}.tar.gz" \ + && local URL_LUA="https://www.lua.org/ftp/lua-${VER_LUA}.tar.gz" \ && echo "Downloading Lua ${VER_LUA} from ${URL_LUA}" \ && install_tar_gz "$URL_LUA" \ - && (cd "/opt/lua-${VER_LUA}" && make linux test && sudo make install "INSTALL_TOP=$LUA_HOME") \ + && (cd "/opt/lua-${VER_LUA}" && sudo make linux test && sudo make install "INSTALL_TOP=$LUA_HOME") \ && sudo ln -sf "$LUA_HOME"/bin/lua* /usr/bin/ \ && sudo rm -rf "/opt/lua-${VER_LUA}" || exit 1 ; type lua && echo "@ Version of Lua installed: $(lua -v)" || exit 1; @@ -380,7 +379,7 @@ setup_bazel() { && [ -n "${VER_BAZEL}" ] \ && local URL_BAZEL="https://github.com/bazelbuild/bazel/releases/download/${VER_BAZEL}/bazel-${VER_BAZEL}-installer-${UNAME}-${ARCH}.sh" \ && curl -o /tmp/bazel.sh -sL "${URL_BAZEL}" && chmod +x /tmp/bazel.sh \ - && /tmp/bazel.sh && rm /tmp/bazel.sh ; + && sudo /tmp/bazel.sh && rm /tmp/bazel.sh ; type bazel && echo "@ Version of bazel: $(bazel --version)" || return 1 ; } @@ -397,8 +396,8 @@ setup_gradle() { fi \ && [ -n "${VER_GRADLE}" ] \ && local URL_GRADLE="https://downloads.gradle.org/distributions/gradle-${VER_GRADLE}-bin.zip" \ - && mv /opt/gradle* /opt/gradle \ - && ln -sf /opt/gradle/bin/gradle /usr/bin ; + && sudo mv /opt/gradle* /opt/gradle \ + && sudo ln -sf /opt/gradle/bin/gradle /usr/bin ; type gradle && echo "@ Version of gradle: $(gradle --version)" || return 1 ; } @@ -418,7 +417,7 @@ setup_yq() { && local URL_YQ="https://github.com/mikefarah/yq/releases/download/v${VER_YQ}/yq_linux_${ARCH}" \ && echo "Installing yq v${VER_YQ} for arch ${ARCH} from: ${URL_YQ}" \ && curl -fSL "${URL_YQ}" -o /tmp/yq \ - && install -m 0755 -D /tmp/yq /opt/bin/yq \ + && sudo install -m 0755 -D /tmp/yq /opt/bin/yq \ && sudo ln -sf /opt/bin/yq /usr/bin/yq \ && rm -f /tmp/yq ; type yq && echo "@ Installed yq: $(yq --version)" ; diff --git a/docker_atom/work/script-setup-db-clients.sh b/docker_atom/work/script-setup-db-clients.sh index 0f1626ac..189b8ecd 100755 --- a/docker_atom/work/script-setup-db-clients.sh +++ b/docker_atom/work/script-setup-db-clients.sh @@ -31,8 +31,8 @@ setup_duckdb() { && echo "Downloading DuckDB ${VER_DUCKDB} from: ${URL_DUCKDB}" \ && curl -L "${URL_DUCKDB}" -o "${TMP_FILE}" \ && if [ "${FILE_DUCKDB##*.}" = zip ]; then unzip -q "${TMP_FILE}" -d /tmp; else tar -xzf "${TMP_FILE}" -C /tmp; fi \ - && mkdir -pv /opt/bin \ - && install -m 0755 /tmp/duckdb /opt/bin/duckdb \ + && sudo mkdir -pv /opt/bin \ + && sudo install -m 0755 /tmp/duckdb /opt/bin/duckdb \ && rm -f "${TMP_FILE}" /tmp/duckdb ; /opt/bin/duckdb --version && echo "@ DuckDB installed" || return 1 ; } @@ -49,7 +49,7 @@ setup_mongosh_client() { # from: https://www.mongodb.com/docs/mongodb-shell/install/ local DISTRO=$(lsb_release -cs) curl -sL https://www.mongodb.org/static/pgp/server-${VER_MONGOSH}.asc | sudo tee /etc/apt/trusted.gpg.d/mongodb.asc - echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu ${DISTRO}/mongodb-org/${VER_MONGOSH} multiverse" > /etc/apt/sources.list.d/mongodb-org-${VER_MONGOSH}.list + echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu ${DISTRO}/mongodb-org/${VER_MONGOSH} multiverse" | sudo tee "/etc/apt/sources.list.d/mongodb-org-${VER_MONGOSH}.list" >/dev/null # will download ~38MB files and use ~218MB disk after installation sudo apt-get update && sudo apt-get -y install mongodb-mongosh type mongosh && echo "@ Version of mongosh client: $(mongosh --version)" || return 1 @@ -66,6 +66,7 @@ setup_redis_client() { setup_oracle_client() { + local ARCH OARCH PKG_ARCH LDAIO PAGE DL BUILD VER ARCH=$(uname -m) case "$ARCH" in x86_64) diff --git a/docker_atom/work/script-setup-net.sh b/docker_atom/work/script-setup-net.sh index 783b878a..9c0f0fe6 100755 --- a/docker_atom/work/script-setup-net.sh +++ b/docker_atom/work/script-setup-net.sh @@ -1,5 +1,6 @@ setup_traefik() { - local VER_TRAEFIK_REQ="${1:-}" \ + local ARCH=$(dpkg --print-architecture) \ + && local VER_TRAEFIK_REQ="${1:-}" \ && local VERS_TRAEFIK=$(curl -sL "https://api.github.com/repos/traefik/traefik/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -vi 'rc' | grep -Po '\d[\d.]+' | sort -rV) \ && if [ -n "${VER_TRAEFIK_REQ}" ]; then local VER_TRAEFIK_RE=${VER_TRAEFIK_REQ#v} \ @@ -9,16 +10,16 @@ setup_traefik() { local VER_TRAEFIK=$(echo "${VERS_TRAEFIK}" | head -1) fi \ && [ -n "${VER_TRAEFIK}" ] \ - && URL_TRAEFIK="https://github.com/traefik/traefik/releases/download/v${VER_TRAEFIK}/traefik_v${VER_TRAEFIK}_linux_$(dpkg --print-architecture).tar.gz" \ + && local URL_TRAEFIK="https://github.com/traefik/traefik/releases/download/v${VER_TRAEFIK}/traefik_v${VER_TRAEFIK}_linux_${ARCH}.tar.gz" \ && curl -o /tmp/TMP.tgz -sL "${URL_TRAEFIK}" \ - && mkdir -pv /opt/bin && tar -C /opt/bin -xzf /tmp/TMP.tgz traefik && rm /tmp/TMP.tgz \ - && chmod +x /opt/bin/traefik && ln -sf /opt/bin/traefik /usr/bin/ ; + && sudo mkdir -pv /opt/bin && sudo tar -C /opt/bin -xzf /tmp/TMP.tgz traefik && rm /tmp/TMP.tgz \ + && sudo chmod +x /opt/bin/traefik && sudo ln -sf /opt/bin/traefik /usr/bin/ ; type traefik && echo "@ Version of traefik: $(traefik version)" || return 1 ; } setup_caddy() { - UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ - && ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ + local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ + && local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ && local VER_CADDY_REQ="${1:-}" \ && local VERS_CADDY=$(curl -sL "https://api.github.com/repos/caddyserver/caddy/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -viE '(beta|rc)' | grep -Po '(\d[\d|.]+)' | sort -rV) \ && if [ -n "${VER_CADDY_REQ}" ]; then @@ -29,10 +30,10 @@ setup_caddy() { local VER_CADDY=$(echo "${VERS_CADDY}" | head -1) fi \ && [ -n "${VER_CADDY}" ] \ - && URL_CADDY="https://github.com/caddyserver/caddy/releases/download/v${VER_CADDY}/caddy_${VER_CADDY}_${UNAME}_${ARCH}.tar.gz" \ + && local URL_CADDY="https://github.com/caddyserver/caddy/releases/download/v${VER_CADDY}/caddy_${VER_CADDY}_${UNAME}_${ARCH}.tar.gz" \ && echo "Downloading Caddy ${VER_CADDY} from ${URL_CADDY}" \ && curl -o /tmp/TMP.tgz -sL "${URL_CADDY}" && tar -C /tmp/ -xzf /tmp/TMP.tgz && rm /tmp/TMP.tgz \ - && mkdir -pv /opt/bin/ && mv /tmp/caddy /opt/bin/ && ln -sf /opt/bin/caddy /usr/local/bin/ ; + && sudo mkdir -pv /opt/bin/ && sudo mv /tmp/caddy /opt/bin/ && sudo ln -sf /opt/bin/caddy /usr/local/bin/ ; type caddy && echo "@ Version of caddy: $(caddy version)" || return 1 ; } @@ -55,8 +56,8 @@ setup_oauth2_proxy() { && echo "Installing oauth2-proxy v${VER_OAUTH2_PROXY} for arch ${ARCH} from: ${URL_OAUTH2_PROXY}" \ && curl -fSL "${URL_OAUTH2_PROXY}" -o /tmp/oauth2-proxy.tar.gz \ && tar -xzf /tmp/oauth2-proxy.tar.gz -C /tmp \ - && install -m 0755 -D "/tmp/oauth2-proxy-v${VER_OAUTH2_PROXY}.linux-${ARCH}/oauth2-proxy" /opt/bin/oauth2-proxy \ - && ln -sf /opt/bin/oauth2-proxy /usr/bin/oauth2-proxy \ + && sudo install -m 0755 -D "/tmp/oauth2-proxy-v${VER_OAUTH2_PROXY}.linux-${ARCH}/oauth2-proxy" /opt/bin/oauth2-proxy \ + && sudo ln -sf /opt/bin/oauth2-proxy /usr/bin/oauth2-proxy \ && rm -rf /tmp/oauth2-proxy* ; type oauth2-proxy && echo "@ Installed oauth2-proxy: $(oauth2-proxy --version)" ; } diff --git a/docker_atom/work/script-setup-sys.sh b/docker_atom/work/script-setup-sys.sh index c2ec3f3a..2a40b0cc 100755 --- a/docker_atom/work/script-setup-sys.sh +++ b/docker_atom/work/script-setup-sys.sh @@ -2,7 +2,7 @@ setup_tini() { - ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ + local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ && local VER_TINI_REQ="${1:-}" \ && local VERS_TINI=$(curl -sL "https://api.github.com/repos/krallin/tini/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po '\d[\d.]+' | sort -rV) \ && if [ -n "${VER_TINI_REQ}" ]; then @@ -13,9 +13,9 @@ setup_tini() { local VER_TINI=$(echo "${VERS_TINI}" | head -1) fi \ && [ -n "${VER_TINI}" ] \ - && URL_TINI="https://github.com/krallin/tini/releases/download/v${VER_TINI}/tini-${ARCH}" \ + && local URL_TINI="https://github.com/krallin/tini/releases/download/v${VER_TINI}/tini-${ARCH}" \ && echo "Downloading Tini ${VER_TINI} from ${URL_TINI}" \ - && curl -o /usr/bin/tini -sL $URL_TINI && chmod +x /usr/bin/tini ; + && sudo curl -o /usr/bin/tini -sL "$URL_TINI" && sudo chmod +x /usr/bin/tini ; type tini && echo "@ Version of tini: $(tini --version)" || return 1 ; # ref: https://cloud-atlas.readthedocs.io/zh-cn/latest/docker/init/docker_tini.html # to run multi-proces with tini: use a bash script ends with the following code @@ -24,8 +24,8 @@ setup_tini() { setup_supervisord() { - UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ - && ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ + local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ + && local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ && local VER_SUPERVISORD_REQ="${1:-}" \ && local VERS_SUPERVISORD=$(curl -sL "https://api.github.com/repos/LabNow-ai/supervisord/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po '(\d[\d|.]+)' | sort -rV) \ && if [ -n "${VER_SUPERVISORD_REQ}" ]; then @@ -36,19 +36,19 @@ setup_supervisord() { local VER_SUPERVISORD=$(echo "${VERS_SUPERVISORD}" | head -1) fi \ && [ -n "${VER_SUPERVISORD}" ] \ - && URL_SUPERVISORD="https://github.com/LabNow-ai/supervisord/releases/download/v${VER_SUPERVISORD}/supervisord_${VER_SUPERVISORD}_${UNAME}_${ARCH}.tar.gz" \ + && local URL_SUPERVISORD="https://github.com/LabNow-ai/supervisord/releases/download/v${VER_SUPERVISORD}/supervisord_${VER_SUPERVISORD}_${UNAME}_${ARCH}.tar.gz" \ && echo "Downloading Supervisord ${VER_SUPERVISORD} from ${URL_SUPERVISORD}" \ && curl -o /tmp/TMP.tgz -sL $URL_SUPERVISORD && tar -C /tmp/ -xzf /tmp/TMP.tgz && rm /tmp/TMP.tgz \ - && mkdir -pv /opt/bin/ && mv /tmp/supervisord /opt/bin/ && ln -sf /opt/bin/supervisord /usr/local/bin/ ; + && sudo mkdir -pv /opt/bin/ && sudo mv /tmp/supervisord /opt/bin/ && sudo ln -sf /opt/bin/supervisord /usr/local/bin/ ; type supervisord && echo "@ Version of supervisord: $(supervisord version)" || return 1 ; } setup_systemd() { - apt-get -qq update -yq --fix-missing \ - && apt-get -qq install -yq --no-install-recommends systemd systemd-cron \ - && rm -f /lib/systemd/system/systemd*udev* \ - && rm -f /lib/systemd/system/getty.target + sudo apt-get -qq update -yq --fix-missing \ + && sudo apt-get -qq install -yq --no-install-recommends systemd systemd-cron \ + && sudo rm -f /lib/systemd/system/systemd*udev* \ + && sudo rm -f /lib/systemd/system/getty.target # ref: https://cloud-atlas.readthedocs.io/zh_CN/latest/docker/init/docker_systemd.html # ENTRYPOINT [ "/usr/lib/systemd/systemd" ] # CMD [ "log-level=info", "unit=sysinit.target" ] diff --git a/docker_atom/work/script-utils.sh b/docker_atom/work/script-utils.sh index fc91d084..528a585f 100755 --- a/docker_atom/work/script-utils.sh +++ b/docker_atom/work/script-utils.sh @@ -1,56 +1,56 @@ # shell util functions # function to debug, resolve package names from a text file and display. -install_echo() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 printf '%s\n' ; } +install_echo() { cut -d "%" -f 1 "$1" | sed '/^$/d' | xargs -r -n1 printf '%s\n' ; } # function to install apt-get packages from a text file which lists package names (add comments with % char) -install_apt() { apt-get -qq update -yq --fix-missing && apt-get -qq install -yq --no-install-recommends $(cat "$1" | cut -d '%' -f 1) ; } +install_apt() { sudo apt-get -qq update -yq --fix-missing && sudo apt-get -qq install -yq --no-install-recommends $(cut -d '%' -f 1 "$1") ; } # function to install conda packages from a text file which lists package names (add comments with % char) -install_conda() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 conda install -yq ; } -install_mamba() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 mamba install -yq --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" ; } +install_conda() { cut -d "%" -f 1 "$1" | sed '/^$/d' | xargs -r -n1 conda install -yq ; } +install_mamba() { cut -d "%" -f 1 "$1" | sed '/^$/d' | xargs -r -n1 mamba install -yq --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" ; } # function to install python packages with pip from a text file which lists package names (add comments with % char) -install_pip() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 pip install --no-cache-dir --root-user-action=ignore -U --pre ; } +install_pip() { cut -d "%" -f 1 "$1" | sed '/^$/d' | xargs -r -n1 pip install --no-cache-dir --root-user-action=ignore -U --pre ; } # function to install R packages from a text file which lists package names (add comments with % char, use quiet=T to be less verbose) install_R() { R -e "options(Ncpus=4);lapply(scan('$1','c',comment.char='%'),function(x){cat(x,system.time(install.packages(x,clean=T,quiet=T)),'\n')})"; } # function to install go packages with go from a text file which lists package names (add comments with % char) -install_go() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 go get -u ; } +install_go() { cut -d "%" -f 1 "$1" | sed '/^$/d' | xargs -r -n1 go get -u ; } # function to install julia packages from a text file which lists package names (add comments with % char) install_julia() { julia -e "import Pkg; l=filter(x->length(x)>0, [split(i,r\"%| |\t\")[1] for i in readlines(\"$1\")]); Pkg.add(l)" ; } # function to install octave packages with go from a text file which lists package names (add comments with % char) -install_octave() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 -I {} sh -c 'octave --eval "pkg install -forge {}"' ; } +install_octave() { cut -d "%" -f 1 "$1" | sed '/^$/d' | xargs -r -n1 -I {} sh -c 'octave --eval "pkg install -forge {}"' ; } # function to download a ZIP file and unzip it to /opt/ -install_zip() { curl -o /tmp/TMP.zip -sL $1 && unzip -q -d /opt/ /tmp/TMP.zip && rm /tmp/TMP.zip ; } +install_zip() { curl -o /tmp/TMP.zip -sL "$1" && sudo unzip -q -d /opt/ /tmp/TMP.zip && rm /tmp/TMP.zip ; } # function to download a .tar.gz file and unzip it to /opt/, add a second argument to extract only those file -install_tar_gz() { curl -o /tmp/TMP.tgz -sL $1 && tar -C /opt/ -xzf /tmp/TMP.tgz ${2:-} && rm /tmp/TMP.tgz ; } +install_tar_gz() { curl -o /tmp/TMP.tgz -sL "$1" && sudo tar -C /opt/ -xzf /tmp/TMP.tgz ${2:-} && rm /tmp/TMP.tgz ; } # function to download a .tar.bz file and unzip it to /opt/, add a second argument to extract only those file -install_tar_bz() { curl -o /tmp/TMP.tbz -sL $1 && tar -C /opt/ -xjf /tmp/TMP.tbz ${2:-} && rm /tmp/TMP.tbz ; } +install_tar_bz() { curl -o /tmp/TMP.tbz -sL "$1" && sudo tar -C /opt/ -xjf /tmp/TMP.tbz ${2:-} && rm /tmp/TMP.tbz ; } # function to download a .tar.xz file and unzip it to /opt/, add a second argument to extract only those file -install_tar_xz() { curl -o /tmp/TMP.txz -sL $1 && tar -C /opt/ -xJf /tmp/TMP.txz ${2:-} && rm /tmp/TMP.txz ; } +install_tar_xz() { curl -o /tmp/TMP.txz -sL "$1" && sudo tar -C /opt/ -xJf /tmp/TMP.txz ${2:-} && rm /tmp/TMP.txz ; } # function to install java packages from a text file which lists JAR file maven full names (add comments with % char) -install_mvn() { cat $1 | cut -d "%" -f 1 | xargs -r -n1 -I {} mvn dependency:copy -DlocalRepositoryDirectory="/tmp/m2repo" -Djavax.net.ssl.trustStorePassword=changeit -Dartifact="{}" -DoutputDirectory="${2:-}" ; } +install_mvn() { cut -d "%" -f 1 "$1" | xargs -r -n1 -I {} mvn dependency:copy -DlocalRepositoryDirectory="/tmp/m2repo" -Djavax.net.ssl.trustStorePassword=changeit -Dartifact="{}" -DoutputDirectory="${2:-}" ; } # function to clean up install__clean(){ - type apt-get && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* ; + type apt-get && sudo apt-get autoremove -y && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/* ; type npm && npm cache clean --force ; type mamba && mamba clean -ya && rm -rf ~/micromamba ; type conda && conda clean -ya && ( rm -rf "${CONDA_PREFIX:-/opt/conda}"/pkgs/* || true ) ; sudo find "${CONDA_PREFIX:-/opt/conda}"/lib -type f -name "*.py[co]" -delete 2>/dev/null || true ; sudo find "${CONDA_PREFIX:-/opt/conda}"/lib -type d -name "__pycache__" -prune -exec rm -rf {} + 2>/dev/null || true ; - ( rm -rf /usr/share/doc /usr/share/man || true ) ; - ( rm -rf /tmp/.* /tmp/* /var/log/* /var/cache/* /root/.cache /root/.* || true ) && chmod ugo+rwXt /tmp ; + ( sudo rm -rf /usr/share/doc /usr/share/man || true ) ; + ( sudo rm -rf /tmp/.* /tmp/* /var/log/* /var/cache/* /root/.cache /root/.* || true ) && sudo chmod ugo+rwXt /tmp ; ls -alh /root /tmp ; echo "@ System release info:" && cat /etc/*release* ; echo "@ System environment variables:" && printenv | sort ; @@ -74,9 +74,9 @@ list_installed_packages() { } fix_permission() { - GROUP_ID=${1:-0}; shift 1; + local GROUP_ID=${1:-0}; shift 1; for d in "$@"; do - find "${d}" \ + sudo find "${d}" \ ! \( \ -group "${GROUP_ID}" \ -a -perm -g+rwX \ @@ -84,7 +84,7 @@ fix_permission() { -exec chgrp "${GROUP_ID}" -- {} \+ \ -exec chmod g+rwX -- {} \+ # setuid, setgid *on directories only* - find "${d}" \ + sudo find "${d}" \ \( \ -type d \ -a ! -perm -6000 \ diff --git a/docker_core/work/script-setup-R.sh b/docker_core/work/script-setup-R.sh index 2571669a..4474b95d 100755 --- a/docker_core/work/script-setup-R.sh +++ b/docker_core/work/script-setup-R.sh @@ -1,6 +1,6 @@ setup_R_datascience() { - apt-get -qq update -yq --fix-missing \ - && apt-get -qq install -yq --no-install-recommends $(cat /opt/utils/install_list_R_datascience.apt | cut -d '%' -f 1) ; + sudo apt-get -qq update -yq --fix-missing \ + && sudo apt-get -qq install -yq --no-install-recommends $(cut -d '%' -f 1 /opt/utils/install_list_R_datascience.apt) ; - R -e "options(Ncpus=4);lapply(scan('/opt/utils/install_list_R_datascience.R','c',comment.char='%'),function(x){cat(x,system.time(install.packages(x,clean=T,quiet=T)),'\n')})"; + sudo R -e "options(Ncpus=4);lapply(scan('/opt/utils/install_list_R_datascience.R','c',comment.char='%'),function(x){cat(x,system.time(install.packages(x,clean=T,quiet=T)),'\n')})"; } diff --git a/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext-mirror.sh b/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext-mirror.sh index d0c69b5c..84587de8 100644 --- a/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext-mirror.sh +++ b/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext-mirror.sh @@ -3,6 +3,7 @@ export DISTRO_CODE_NAME=$(lsb_release -cs) APT_KEYRING_DIR="/etc/apt/keyrings" APT_SOURCE_DIR="/etc/apt/sources.list.d" +sudo mkdir -p "$APT_KEYRING_DIR" "$APT_SOURCE_DIR" add_apt_source() { local key_url="$1" @@ -12,8 +13,8 @@ add_apt_source() { local keyring_path="${APT_KEYRING_DIR}/${keyring_name}" local list_path="${APT_SOURCE_DIR}/${list_name}" - curl -fsSL "$key_url" | gpg --dearmor > "$keyring_path" - curl -fsSL "$list_url" > "$list_path" + curl -fsSL "$key_url" | gpg --dearmor | sudo tee "$keyring_path" >/dev/null + curl -fsSL "$list_url" | sudo tee "$list_path" >/dev/null } diff --git a/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext.sh b/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext.sh index 62e125e8..236ad5dd 100644 --- a/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext.sh +++ b/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext.sh @@ -1,14 +1,14 @@ if [ ${PG_MAJOR} != "17" ]; then # Fix pgagent for PG 15 / 16 - rm -rf /usr/share/postgresql/${PG_MAJOR}/extension/pgagent* - mv /usr/share/postgresql/17/extension/pgagent* /usr/share/postgresql/${PG_MAJOR}/extension/ + sudo rm -rf /usr/share/postgresql/${PG_MAJOR}/extension/pgagent* + sudo mv /usr/share/postgresql/17/extension/pgagent* /usr/share/postgresql/${PG_MAJOR}/extension/ fi -mv /var/lib/postgresql/extension/* /usr/share/postgresql/${PG_MAJOR}/extension/ +sudo mv /var/lib/postgresql/extension/* /usr/share/postgresql/${PG_MAJOR}/extension/ ## remove un-used PG versions and extensions -find /usr/lib/postgresql -mindepth 1 -maxdepth 1 -type d -not -name "${PG_MAJOR}" -not -name "lib" -exec rm -rf {} \; -find /usr/share/postgresql -mindepth 1 -maxdepth 1 -type d -not -name "${PG_MAJOR}" -exec rm -rf {} \; +sudo find /usr/lib/postgresql -mindepth 1 -maxdepth 1 -type d -not -name "${PG_MAJOR}" -not -name "lib" -exec rm -rf {} \; +sudo find /usr/share/postgresql -mindepth 1 -maxdepth 1 -type d -not -name "${PG_MAJOR}" -exec rm -rf {} \; export USE_PGXS=1 @@ -17,14 +17,14 @@ export USE_PGXS=1 setup_pg_search() { ## ref1: https://docs.paradedb.com/deploy/self-hosted/extension ## ref2: https://github.com/paradedb/paradedb - ARCH="amd64" \ - && VER_PG_SEARCH=$(curl -fsSL "https://api.github.com/repos/paradedb/paradedb/releases?per_page=50" | grep -Po '(?<="tag_name": "v)[^"]+' | grep -v -- '-rc' | head -1) \ - && URL_PG_SEARCH="https://github.com/paradedb/paradedb/releases/download/v${VER_PG_SEARCH}/postgresql-${PG_MAJOR}-pg-search_${VER_PG_SEARCH}-1PARADEDB-$(lsb_release -cs)_amd64.deb" \ + local ARCH="amd64" \ + && local VER_PG_SEARCH=$(curl -fsSL "https://api.github.com/repos/paradedb/paradedb/releases?per_page=50" | grep -Po '(?<="tag_name": "v)[^"]+' | grep -v -- '-rc' | head -1) \ + && local URL_PG_SEARCH="https://github.com/paradedb/paradedb/releases/download/v${VER_PG_SEARCH}/postgresql-${PG_MAJOR}-pg-search_${VER_PG_SEARCH}-1PARADEDB-$(lsb_release -cs)_amd64.deb" \ && echo "Downloading pg_search ${VER_PG_SEARCH} from: ${URL_PG_SEARCH}" \ && mkdir -pv /tmp/pg_search/ && cd /tmp/pg_search \ && curl -fsSL -O "${URL_PG_SEARCH}" \ - && apt-get -qq install -yq --no-install-recommends libopenblas0 \ - && dpkg -i *.deb + && sudo apt-get -qq install -yq --no-install-recommends libopenblas0 \ + && sudo dpkg -i *.deb } @@ -32,18 +32,18 @@ setup_apache_age() { cd /tmp git clone --depth 1 -b PG${PG_MAJOR} https://github.com/apache/age cd /tmp/age - apt-get -qq install -yq --no-install-recommends flex bison postgresql-server-dev-${PG_MAJOR} - make -j8 && make install + sudo apt-get -qq install -yq --no-install-recommends flex bison postgresql-server-dev-${PG_MAJOR} + make -j8 && sudo make install } setup_pgvectorscale() { ## ref: https://github.com/timescale/pgvectorscale - ARCH="amd64" \ - && VER_PGVS=$(curl -sL "https://api.github.com/repos/timescale/pgvectorscale/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[\d.]+' ) \ - && URL_PGVS="https://github.com/timescale/pgvectorscale/releases/download/${VER_PGVS}/pgvectorscale-${VER_PGVS}-pg${PG_MAJOR}-${ARCH}.zip" \ + local ARCH="amd64" \ + && local VER_PGVS=$(curl -sL "https://api.github.com/repos/timescale/pgvectorscale/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[\d.]+' ) \ + && local URL_PGVS="https://github.com/timescale/pgvectorscale/releases/download/${VER_PGVS}/pgvectorscale-${VER_PGVS}-pg${PG_MAJOR}-${ARCH}.zip" \ && mkdir -pv /tmp/pgvectorscale/ && cd /tmp/pgvectorscale \ - && install_zip ${URL_PGVS} && mv /opt/pgvectorscal* /tmp/pgvectorscale/ \ - && dpkg -i *.deb + && install_zip "${URL_PGVS}" && sudo mv /opt/pgvectorscal* /tmp/pgvectorscale/ \ + && sudo dpkg -i *.deb } setup_pgroonga(){ @@ -51,12 +51,12 @@ setup_pgroonga(){ ## ref2: https://github.com/pgroonga/docker mkdir -pv /tmp/pgroonga && cd /tmp/pgroonga curl -fsSL -O https://apache.jfrog.io/artifactory/arrow/debian/apache-arrow-apt-source-latest-$(lsb_release -cs).deb \ - && apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release -cs).deb \ + && sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release -cs).deb \ && rm apache-arrow-apt-source-latest-$(lsb_release -cs).deb \ && curl -fsSL -O https://packages.groonga.org/debian/groonga-apt-source-latest-$(lsb_release -cs).deb \ - && apt install -y -V ./groonga-apt-source-latest-$(lsb_release -cs).deb \ + && sudo apt install -y -V ./groonga-apt-source-latest-$(lsb_release -cs).deb \ && rm groonga-apt-source-latest-$(lsb_release -cs).deb \ - && apt update && apt install -y -V \ + && sudo apt update && sudo apt install -y -V \ postgresql-${PG_MAJOR}-pgdg-pgroonga groonga-normalizer-mysql groonga-token-filter-stem groonga-tokenizer-mecab } @@ -67,7 +67,7 @@ setup_pg_duckdb() { cd pg_duckdb # TODO: temp remove this pin once pg_duckdb updates its httpfs revision for duckdb-httpfs#517. sed -i '/GIT_URL https:\/\/github.com\/duckdb\/duckdb-httpfs/{n;s/GIT_TAG .*/GIT_TAG 3b48554efa84bdf4397f4758f9701646c65d06c1/;}' third_party/pg_duckdb_extensions.cmake - apt-get -qq install -yq --no-install-recommends postgresql-server-dev-${PG_MAJOR} liblz4-dev libcurl4-openssl-dev ninja-build + sudo apt-get -qq install -yq --no-install-recommends postgresql-server-dev-${PG_MAJOR} liblz4-dev libcurl4-openssl-dev ninja-build make -j8 && sudo make install } @@ -75,6 +75,6 @@ setup_pg_net() { cd /tmp git clone --depth 1 -b master https://github.com/supabase/pg_net cd /tmp/pg_net - apt-get -qq install -yq --no-install-recommends libcurl4-gnutls-dev postgresql-server-dev-${PG_MAJOR} + sudo apt-get -qq install -yq --no-install-recommends libcurl4-gnutls-dev postgresql-server-dev-${PG_MAJOR} make -j8 && sudo make install } diff --git a/docker_docker_kit/work/script-setup-docker.sh b/docker_docker_kit/work/script-setup-docker.sh index 8f9a5520..96cee65c 100755 --- a/docker_docker_kit/work/script-setup-docker.sh +++ b/docker_docker_kit/work/script-setup-docker.sh @@ -2,9 +2,9 @@ setup_docker_compose() { - ARCH=$(uname -m) \ - && VER_COMPOSE=$(curl -sL "https://api.github.com/repos/docker/compose/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[.\d]+') \ - && URL_COMPOSE="https://github.com/docker/compose/releases/download/v${VER_COMPOSE}/docker-compose-linux-${ARCH}" \ + local ARCH=$(uname -m) \ + && local VER_COMPOSE=$(curl -sL "https://api.github.com/repos/docker/compose/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[.\d]+') \ + && local URL_COMPOSE="https://github.com/docker/compose/releases/download/v${VER_COMPOSE}/docker-compose-linux-${ARCH}" \ && echo "Downloading Compose from: ${URL_COMPOSE}" \ && sudo curl -o /usr/bin/docker-compose -sL ${URL_COMPOSE} \ && sudo chmod +x /usr/bin/docker-compose ; @@ -12,9 +12,9 @@ setup_docker_compose() { } setup_docker_syncer() { - ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ - && VER_SYNCER="$(curl -sL "https://api.github.com/repos/AliyunContainerService/image-syncer/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[.\d]+')" \ - && URL_SYNCER="https://github.com/AliyunContainerService/image-syncer/releases/download/v${VER_SYNCER}/image-syncer-v${VER_SYNCER}-linux-${ARCH}.tar.gz" \ + local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ + && local VER_SYNCER="$(curl -sL "https://api.github.com/repos/AliyunContainerService/image-syncer/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[.\d]+')" \ + && local URL_SYNCER="https://github.com/AliyunContainerService/image-syncer/releases/download/v${VER_SYNCER}/image-syncer-v${VER_SYNCER}-linux-${ARCH}.tar.gz" \ && echo "Downloading image-syncer from: ${URL_SYNCER}" \ && curl -o /tmp/image_syncer.tgz -sL ${URL_SYNCER} \ && mkdir -pv /tmp/image_syncer && tar -zxvf /tmp/image_syncer.tgz -C /tmp/image_syncer \ diff --git a/docker_docker_kit/work/script-sync-image.sh b/docker_docker_kit/work/script-sync-image.sh index e2584174..9453666d 100644 --- a/docker_docker_kit/work/script-sync-image.sh +++ b/docker_docker_kit/work/script-sync-image.sh @@ -23,51 +23,30 @@ registries_target=${DOCKER_MIRROR_REGISTRY:-} image_single= append_target_registries() { - if [ -n "$registries_target" ]; then - registries_target="${registries_target},$1" - else - registries_target=$1 - fi + registries_target="${registries_target}${registries_target:+,}$1" +} + +read_option_value() { + case "$2" in + *=*) option_value=${2#*=}; option_shift=1 ;; + *) + [ "$#" -ge 3 ] || { echo "Missing value for $1" >&2; exit 2; } + option_value=$3 + option_shift=2 + ;; + esac } while [ "$#" -gt 0 ]; do case "$1" in - --file-workflow|--workflow-file) - [ "$#" -ge 2 ] || { echo "Missing value for $1" >&2; exit 2; } - file_workflow=$2 - shift 2 - ;; - --file-workflow=*|--workflow-file=*) - file_workflow=${1#*=} - shift - ;; - --image-namespace-src|--image-namespace) - [ "$#" -ge 2 ] || { echo "Missing value for $1" >&2; exit 2; } - namespace_img_src=$2 - shift 2 - ;; - --image-namespace-src=*|--image-namespace=*) - namespace_img_src=${1#*=} - shift - ;; - --image-namespace-dst) - [ "$#" -ge 2 ] || { echo "Missing value for --image-namespace-dst" >&2; exit 2; } - namespace_img_dst=$2 - shift 2 - ;; - --image-namespace-dst=*) - namespace_img_dst=${1#*=} - shift - ;; - --source-registry) - [ "$#" -ge 2 ] || { echo "Missing value for $1" >&2; exit 2; } - registry_source=$2 - shift 2 - ;; - --source-registry=*) - registry_source=${1#*=} - shift - ;; + --file-workflow|--workflow-file|--file-workflow=*|--workflow-file=*) + read_option_value "$1" "$@"; file_workflow=$option_value; shift "$option_shift" ;; + --image-namespace-src|--image-namespace|--image-namespace-src=*|--image-namespace=*) + read_option_value "$1" "$@"; namespace_img_src=$option_value; shift "$option_shift" ;; + --image-namespace-dst|--image-namespace-dst=*) + read_option_value "$1" "$@"; namespace_img_dst=$option_value; shift "$option_shift" ;; + --source-registry|--source-registry=*) + read_option_value "$1" "$@"; registry_source=$option_value; shift "$option_shift" ;; --target-registry) shift target_found=false @@ -82,24 +61,10 @@ while [ "$#" -gt 0 ]; do append_target_registries "${1#*=}" shift ;; - --file-auth|--auth-file) - [ "$#" -ge 2 ] || { echo "Missing value for $1" >&2; exit 2; } - file_auth=$2 - shift 2 - ;; - --file-auth=*|--auth-file=*) - file_auth=${1#*=} - shift - ;; - --image) - [ "$#" -ge 2 ] || { echo "Missing value for $1" >&2; exit 2; } - image_single=$2 - shift 2 - ;; - --image=*) - image_single=${1#*=} - shift - ;; + --file-auth|--auth-file|--file-auth=*|--auth-file=*) + read_option_value "$1" "$@"; file_auth=$option_value; shift "$option_shift" ;; + --image|--image=*) + read_option_value "$1" "$@"; image_single=$option_value; shift "$option_shift" ;; --help|-h) usage exit 0 @@ -145,9 +110,7 @@ else echo "Failed to parse workflow file: $file_workflow" >&2 exit 1 fi - while IFS= read -r name_job || [ -n "$name_job" ]; do - printf '%s\n' "$name_job" | tr ',' '\n' >> "$file_names" - done < "$file_names_raw" + tr ',' '\n' < "$file_names_raw" > "$file_names" fi printf '%s\n' "$registries_target" | tr ',' '\n' | @@ -157,13 +120,18 @@ if [ ! -s "$file_targets" ]; then fi status_total=0 +run_image_sync() { + image-syncer "$@" + status_sync=$? + status_total=$((status_total + status_sync)) +} + while IFS= read -r image_name || [ -n "$image_name" ]; do + path_image_src=$image_name + path_image_dst=$image_name if [ -z "$image_single" ]; then path_image_src=$namespace_img_src/$image_name path_image_dst=$namespace_img_dst/$image_name - else - path_image_src=$image_name - path_image_dst=$image_name fi image_source=$registry_source/$path_image_src echo "Docker image sync job name found: $path_image_src" @@ -179,14 +147,12 @@ while IFS= read -r image_name || [ -n "$image_name" ]; do fi done < "$file_targets" export TARGET_REGISTRIES - if ! yq -n -o=json '{"images": {(strenv(SOURCE_IMAGE)): (strenv(TARGET_REGISTRIES) | split(","))}}' > "$file_config"; then + if ! yq -n -o=json '{(strenv(SOURCE_IMAGE)): (strenv(TARGET_REGISTRIES) | split(","))}' > "$file_config"; then echo "Failed to generate image config for $path_image_src" >&2 status_total=$((status_total + 1)) continue fi - image-syncer --proc=16 --retries=2 "--auth=$file_auth" "--images=$file_config" - status_sync=$? - status_total=$((status_total + status_sync)) + run_image_sync --proc=16 --retries=2 "--auth=$file_auth" "--images=$file_config" else if [ -z "${DOCKER_MIRROR_REGISTRY_USERNAME:-}" ] || [ -z "${DOCKER_MIRROR_REGISTRY_PASSWORD:-}" ]; then echo "DOCKER_MIRROR_REGISTRY_USERNAME and DOCKER_MIRROR_REGISTRY_PASSWORD are required without --file-auth" >&2 @@ -206,9 +172,7 @@ while IFS= read -r image_name || [ -n "$image_name" ]; do status_total=$((status_total + 1)) continue fi - image-syncer --proc=16 --retries=2 "--config=$file_config" - status_sync=$? - status_total=$((status_total + status_sync)) + run_image_sync --proc=16 --retries=2 "--config=$file_config" done < "$file_targets" fi done < "$file_names" diff --git a/tool.sh b/tool.sh index f76e39eb..838afbf9 100755 --- a/tool.sh +++ b/tool.sh @@ -13,12 +13,12 @@ else fi -CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/') +CI_PROJECT_SPACE=${CI_PROJECT_BRANCH%%/*} # If on the main branch, image namespace will be same as CI_PROJECT_NAME's name space; # else (not main branch), image namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE}. [ "${CI_PROJECT_BRANCH}" = "main" ] && NAMESPACE_SUFFIX="" || NAMESPACE_SUFFIX="0${CI_PROJECT_SPACE}" ; -export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})${NAMESPACE_SUFFIX}" ; +export CI_PROJECT_NAMESPACE="$(dirname "${CI_PROJECT_NAME}")${NAMESPACE_SUFFIX}" ; export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') export IMG_NAMESPACE_SRC=$(echo "${IMG_NAMESPACE_SRC:-$IMG_NAMESPACE}" | awk '{print tolower($0)}') @@ -37,7 +37,7 @@ echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}" build_image() { echo "$@" ; - IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)"; + local IMG=$1 TAG=$2 FILE=$3; shift 3; local VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX} WORKDIR="$(dirname "$FILE")"; BUILDKIT_PROGRESS=plain DOCKER_BUILDKIT=1 \ docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" \ --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" @@ -47,7 +47,7 @@ build_image() { build_image_no_tag() { echo "$@" ; - IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)"; + local IMG=$1 TAG=$2 FILE=$3; shift 3; local WORKDIR="$(dirname "$FILE")"; BUILDKIT_PROGRESS=plain DOCKER_BUILDKIT=1 \ docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" \ --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" @@ -55,13 +55,13 @@ build_image_no_tag() { } alias_image() { - IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; + local IMG_1=$1 TAG_1=$2 IMG_2=$3 TAG_2=$4; shift 4; local VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; docker tag "${IMG_PREFIX_DST}/${IMG_1}:${TAG_1}" "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" docker tag "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" "${IMG_PREFIX_DST}/${IMG_2}:${VER}" } push_image() { - KEYWORD="${1:-second}"; + local KEYWORD="${1:-second}" IMAGES IMG status; docker image prune --force && docker images; IMAGES=$(docker images --format "{{.Repository}}\t{{.Tag}}\t{{.CreatedSince}}" | grep "${IMG_PREFIX_DST}" | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; [ -n "${IMAGES}" ] || { echo "!! No images matched keyword: ${KEYWORD}"; return 1; } @@ -75,9 +75,10 @@ push_image() { } clear_images() { - KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images built days ago - IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; - IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ; + local KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images built days ago + local IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; + local IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ; + local IMG status for IMG in $(echo "$IMGS_1 $IMGS_2" | tr " " "\n") ; do docker rmi "${IMG}" ; status=$?; echo "[${status}] image removed > ${IMG}"; From 5077130d1f42cf525853779be5f08d88eb7a7c77 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Sat, 14 Nov 2026 21:02:16 +0000 Subject: [PATCH 5/8] debug scripts --- .../work/localize/run-config-mirror-cernet.sh | 2 +- docker_atom/work/script-setup-core.sh | 15 +++++++++++---- docker_atom/work/script-utils.sh | 6 +++--- .../docker-entrypoint-initdb.d/01-init-db.sh | 1 + docker_docker_kit/work/script-sync-image.sh | 5 +++-- tool.sh | 1 + 6 files changed, 20 insertions(+), 10 deletions(-) diff --git a/docker_atom/work/localize/run-config-mirror-cernet.sh b/docker_atom/work/localize/run-config-mirror-cernet.sh index 672d2acc..f47584c3 100755 --- a/docker_atom/work/localize/run-config-mirror-cernet.sh +++ b/docker_atom/work/localize/run-config-mirror-cernet.sh @@ -10,7 +10,7 @@ echo "Setup timezone, current date: $(date)" OS_ID=$(sed -n 's/^ID=//p' /etc/os-release | tr -d '"') echo "Found ${OS_ID} system, setting mirror for ${OS_ID}" -FILE_DEB=$([ -f /etc/apt/sources.list.d/${OS_ID}.sources ] && echo /etc/apt/sources.list.d/${OS_ID}.sources || echo /etc/apt/sources.list) +FILE_DEB=$([ -f "/etc/apt/sources.list.d/${OS_ID}.sources" ] && echo "/etc/apt/sources.list.d/${OS_ID}.sources" || echo /etc/apt/sources.list) if [ -f "$FILE_DEB" ]; then sudo sed -i 's|mirrors.*.com/ubuntu|mirrors.cernet.edu.cn/ubuntu|g' "$FILE_DEB" sudo sed -i 's|archive.ubuntu.com/ubuntu|mirrors.cernet.edu.cn/ubuntu|g' "$FILE_DEB" diff --git a/docker_atom/work/script-setup-core.sh b/docker_atom/work/script-setup-core.sh index e182c8da..85506e18 100755 --- a/docker_atom/work/script-setup-core.sh +++ b/docker_atom/work/script-setup-core.sh @@ -55,7 +55,7 @@ EOF && sudo conda update --all --quiet --yes ; # remove non-necessary folder/symlink "python3.1" exists - rm -rf "${CONDA_PREFIX}"/bin/python3.1 "${CONDA_PREFIX}"/lib/python3.1 ; + sudo rm -rf "${CONDA_PREFIX}"/bin/python3.1 "${CONDA_PREFIX}"/lib/python3.1 ; # These conda pkgs shouldn't be removed (otherwise will cause RemoveError) since they are directly required by conda: pip setuptools pycosat pyopenssl requests ruamel_yaml # CONDA_PY_PKGS=$(conda list | grep "py3" | cut -d " " -f 1 | sed "/#/d;/conda/d;/pip/d;/setuptools/d;/pycosat/d;/pyopenssl/d;/requests/d;/ruamel_yaml/d;") \ @@ -131,10 +131,17 @@ setup_java_base() { local URL_JDK_adoptium="https://api.github.com/repos/adoptium/temurin${VER_JDK_MAJOR}-binaries/releases/latest" \ && local URL_JDK_DOWNLOAD=$( curl -sL "$URL_JDK_adoptium" | grep 'tar.gz' | grep -vE '.sha256|.sig|.json|debug|test' | grep "${ARCH}" | grep -i "$(uname)" \ - | grep -oP "(https?://[^\s<>\'\"]*)" | grep -E "$(if [ "$IS_ALPINE" = true ]; then echo 'alpine'; else echo -v 'alpine'; fi)" | head -n1 + | grep -oP "(https?://[^\s<>\'\"]*)" \ + | if [ "$IS_ALPINE" = true ]; then grep -E 'alpine'; else grep -v 'alpine'; fi \ + | head -n1 ) ; fi + if [ -z "$URL_JDK_DOWNLOAD" ]; then + echo "Failed to find a JDK ${VER_JDK_MAJOR} download for ${ARCH} (Alpine: ${IS_ALPINE})" >&2 + return 1 + fi + echo "Installing JDK version ${VER_JDK} from: ${URL_JDK_DOWNLOAD}" ; install_tar_gz "${URL_JDK_DOWNLOAD}" && sudo mv /opt/jdk* /opt/jdk && sudo ln -sf /opt/jdk/bin/* /usr/bin/ ; type java && echo "@ Version of Java (java): $(java -version)" || return 1 ; @@ -309,7 +316,7 @@ setup_julia() { && local VER_JULIA_MAJOR=$(echo "${VER_JULIA}" | cut -d '.' -f1,2) \ && local URL_JULIA="https://julialang-s3.julialang.org/bin/linux/${ARCH_2}/${VER_JULIA_MAJOR}/julia-${VER_JULIA}-linux-${ARCH_1}.tar.gz" \ && echo "Downloading Julia version ${VER_JULIA} from: ${URL_JULIA}" \ - && install_tar_gz $URL_JULIA \ + && install_tar_gz "$URL_JULIA" \ && sudo mv /opt/julia-* /opt/julia \ && sudo ln -fs /opt/julia/bin/julia /usr/bin/julia \ && sudo mkdir -pv /opt/julia/pkg \ @@ -355,7 +362,7 @@ setup_lua_rocks() { && [ -n "${VER_LUA_ROCKS}" ] \ && local URL_LUA_ROCKS="https://luarocks.org/releases/luarocks-${VER_LUA_ROCKS}.tar.gz" \ && echo "Downloading luarocks ${VER_LUA_ROCKS} from ${URL_LUA_ROCKS}" \ - && install_tar_gz $URL_LUA_ROCKS \ + && install_tar_gz "$URL_LUA_ROCKS" \ && sudo mv /opt/luarocks-* /tmp/luarocks && cd /tmp/luarocks \ && sudo ./configure --prefix=${LUA_HOME:-"/opt/lua"} --with-lua-include=${LUA_HOME:-"/opt/lua"}/include && sudo make install \ && sudo ln -sf /opt/lua/bin/lua* /usr/bin/ \ diff --git a/docker_atom/work/script-utils.sh b/docker_atom/work/script-utils.sh index 528a585f..32ef1b25 100755 --- a/docker_atom/work/script-utils.sh +++ b/docker_atom/work/script-utils.sh @@ -44,8 +44,8 @@ install_mvn() { cut -d "%" -f 1 "$1" | xargs -r -n1 -I {} mvn dependency:copy -D install__clean(){ type apt-get && sudo apt-get autoremove -y && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/* ; type npm && npm cache clean --force ; - type mamba && mamba clean -ya && rm -rf ~/micromamba ; - type conda && conda clean -ya && ( rm -rf "${CONDA_PREFIX:-/opt/conda}"/pkgs/* || true ) ; + type mamba && sudo mamba clean -ya && rm -rf ~/micromamba ; + type conda && sudo conda clean -ya && ( sudo rm -rf "${CONDA_PREFIX:-/opt/conda}"/pkgs/* || true ) ; sudo find "${CONDA_PREFIX:-/opt/conda}"/lib -type f -name "*.py[co]" -delete 2>/dev/null || true ; sudo find "${CONDA_PREFIX:-/opt/conda}"/lib -type d -name "__pycache__" -prune -exec rm -rf {} + 2>/dev/null || true ; @@ -74,7 +74,7 @@ list_installed_packages() { } fix_permission() { - local GROUP_ID=${1:-0}; shift 1; + local GROUP_ID=${1:-0} d; shift 1; for d in "$@"; do sudo find "${d}" \ ! \( \ diff --git a/docker_db_postgres/rootfs/docker-entrypoint-initdb.d/01-init-db.sh b/docker_db_postgres/rootfs/docker-entrypoint-initdb.d/01-init-db.sh index dd47a7b4..a219b092 100644 --- a/docker_db_postgres/rootfs/docker-entrypoint-initdb.d/01-init-db.sh +++ b/docker_db_postgres/rootfs/docker-entrypoint-initdb.d/01-init-db.sh @@ -36,6 +36,7 @@ cat ${PGDATA}/conf.d/* # docker_temp_server_stop && sleep 2s && docker_temp_server_start enable_all_extensions() { + local e psql "$@" -At -c "SELECT name FROM pg_available_extensions WHERE name NOT IN (SELECT extname FROM pg_extension)" | while read e; do psql "$@" -c "CREATE EXTENSION IF NOT EXISTS \"$e\" CASCADE" >/dev/null || echo "Skip $e"; done } diff --git a/docker_docker_kit/work/script-sync-image.sh b/docker_docker_kit/work/script-sync-image.sh index 9453666d..5c73d9ce 100644 --- a/docker_docker_kit/work/script-sync-image.sh +++ b/docker_docker_kit/work/script-sync-image.sh @@ -21,6 +21,8 @@ namespace_img_dst=${IMG_NAMESPACE_DST:-} registry_source=${REGISTRY_URL:-docker.io} registries_target=${DOCKER_MIRROR_REGISTRY:-} image_single= +option_value= +option_shift= append_target_registries() { registries_target="${registries_target}${registries_target:+,}$1" @@ -122,8 +124,7 @@ fi status_total=0 run_image_sync() { image-syncer "$@" - status_sync=$? - status_total=$((status_total + status_sync)) + status_total=$((status_total + $?)) } while IFS= read -r image_name || [ -n "$image_name" ]; do diff --git a/tool.sh b/tool.sh index 838afbf9..8364b9da 100755 --- a/tool.sh +++ b/tool.sh @@ -88,6 +88,7 @@ clear_images() { remove_folder() { + local dir for dir in "$@"; do if [ -d "$dir" ]; then echo "Removing folder: $dir" ; From 9b1114d98fc6ccbc34560536001d491b66cab6b9 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Sun, 15 Nov 2026 07:31:19 +0000 Subject: [PATCH 6/8] update --- docker_atom/work/script-setup-core.sh | 46 +++++++++---------- docker_atom/work/script-setup-db-clients.sh | 14 +++--- docker_atom/work/script-setup-net.sh | 14 +++--- docker_atom/work/script-setup-sys.sh | 8 ++-- docker_atom/work/script-utils.sh | 14 +++--- docker_cuda/nvidia-cuda.Dockerfile | 2 +- .../opt/utils/script-setup-pg-ext-mirror.sh | 4 +- .../rootfs/opt/utils/script-setup-pg-ext.sh | 12 ++--- docker_docker_kit/work/script-setup-docker.sh | 8 ++-- 9 files changed, 63 insertions(+), 59 deletions(-) diff --git a/docker_atom/work/script-setup-core.sh b/docker_atom/work/script-setup-core.sh index 85506e18..3bb3e7fc 100755 --- a/docker_atom/work/script-setup-core.sh +++ b/docker_atom/work/script-setup-core.sh @@ -6,7 +6,7 @@ setup_mamba() { local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && local ARCH=$(uname -m | sed -e 's/x86_64/64/') \ && local VER_MICROMAMBA_REQ="${1:-}" \ - && local VERS_MICROMAMBA=$(curl -fsSL "https://api.github.com/repos/mamba-org/micromamba-releases/releases?per_page=50" | grep -oP '"tag_name":\s*"\K[^"]+' | grep -viE '(alpha|beta|rc)' | sort -rV) \ + && local VERS_MICROMAMBA=$(curl_fsSL "https://api.github.com/repos/mamba-org/micromamba-releases/releases?per_page=50" | grep -oP '"tag_name":\s*"\K[^"]+' | grep -viE '(alpha|beta|rc)' | sort -rV) \ && if [ -n "${VER_MICROMAMBA_REQ}" ]; then local VER_MICROMAMBA_RE=${VER_MICROMAMBA_REQ#v} \ && VER_MICROMAMBA_RE=$(printf '%s' "$VER_MICROMAMBA_RE" | sed 's/[.]/\\./g') \ @@ -78,7 +78,7 @@ setup_conda_with_mamba() { setup_conda_download() { ## https://docs.conda.io/projects/miniconda/en/latest/index.html local URL_CONDA="https://repo.continuum.io/miniconda/Miniconda3-latest-$(uname)-$(arch).sh" \ - && curl -sL "$URL_CONDA" -o /tmp/conda.sh \ + && curl_fsSL "$URL_CONDA" -o /tmp/conda.sh \ && sudo mkdir -pv "${CONDA_PREFIX}" && sudo bash /tmp/conda.sh -f -b -p "${CONDA_PREFIX}/" \ && rm -rf /tmp/conda.sh \ && setup_conda_postprocess ; @@ -86,7 +86,7 @@ setup_conda_download() { setup_nvtop() { ## The compilation requires CMake 3.18 or higher, while the default version in CUDA 11.2 images is 3.16.3 - curl -sL https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \ + curl_fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo gpg --dearmor -o /usr/share/keyrings/kitware-archive-keyring.gpg \ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/kitware.list \ && sudo apt-get -qq update -yq --fix-missing && sudo apt-get -qq install -yq --no-install-recommends cmake ; @@ -119,7 +119,7 @@ setup_java_base() { local VER_JDK="$VER_JDK_MAJOR" ; if [ "$VER_JDK_MAJOR" = "latest" ] || { case "$VER_JDK_MAJOR" in ''|*[!0-9]*) false ;; *) [ "$VER_JDK_MAJOR" -gt 20 ] ;; esac; }; then local PAGE_JDK_DOWNLOAD="https://www.oracle.com/java/technologies/downloads/" - local PAGE_JDK=$(curl -sL --fail "$PAGE_JDK_DOWNLOAD" || { echo "Failed to fetch Oracle JDK download page" && return 1; } ) + local PAGE_JDK=$(curl_fsSL "$PAGE_JDK_DOWNLOAD" || { echo "Failed to fetch Oracle JDK download page" && return 1; } ) if [ "$VER_JDK_MAJOR" = "latest" ]; then VER_JDK=$(echo "$PAGE_JDK" | grep -oE 'jdk-[0-9]+' | sed 's/jdk-//' | sort -rV | head -1) ; @@ -130,7 +130,7 @@ setup_java_base() { else local URL_JDK_adoptium="https://api.github.com/repos/adoptium/temurin${VER_JDK_MAJOR}-binaries/releases/latest" \ && local URL_JDK_DOWNLOAD=$( - curl -sL "$URL_JDK_adoptium" | grep 'tar.gz' | grep -vE '.sha256|.sig|.json|debug|test' | grep "${ARCH}" | grep -i "$(uname)" \ + curl_fsSL "$URL_JDK_adoptium" | grep 'tar.gz' | grep -vE '.sha256|.sig|.json|debug|test' | grep "${ARCH}" | grep -i "$(uname)" \ | grep -oP "(https?://[^\s<>\'\"]*)" \ | if [ "$IS_ALPINE" = true ]; then grep -E 'alpine'; else grep -v 'alpine'; fi \ | head -n1 @@ -151,7 +151,7 @@ setup_java_base() { setup_java_maven() { local VER_MAVEN="${1:-3}" \ - && local ATOM_MAVEN=$(curl -sL https://maven.apache.org/docs/history.html | grep '/ref/') \ + && local ATOM_MAVEN=$(curl_fsSL https://maven.apache.org/docs/history.html | grep '/ref/') \ && local VERS_MAVEN=$(echo "${ATOM_MAVEN}" | grep -oP '(?<=/ref/)[0-9]+\.[0-9]+(\.[0-9]+)?(-[a-z]+-[0-9]+)?(?=/)' | grep -viE '(alpha|beta|rc)' | sort -rV) \ && local VER_MAVEN_RE=${VER_MAVEN#v} \ && VER_MAVEN_RE=$(printf '%s' "$VER_MAVEN_RE" | sed 's/[.]/\\./g') \ @@ -170,7 +170,7 @@ setup_node_base() { local VER_NODEJS_REQ="${1:-}" \ && local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && local ARCH=$(uname -m | sed -e 's/x86_64/x64/' -e 's/aarch64/arm64/') \ - && local VERS_NODEJS=$(curl -sL "https://api.github.com/repos/nodejs/node/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -vi alpha | grep -Po '\d[\d.]+' | sort -rV) \ + && local VERS_NODEJS=$(curl_fsSL "https://api.github.com/repos/nodejs/node/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -vi alpha | grep -Po '\d[\d.]+' | sort -rV) \ && if [ -n "${VER_NODEJS_REQ}" ]; then local VER_NODEJS_RE=${VER_NODEJS_REQ#v} && VER_NODEJS_RE=$(printf '%s' "$VER_NODEJS_RE" | sed 's/[.]/\\./g') \ && local VER_NODEJS=$(echo "${VERS_NODEJS}" | grep -m1 -E "^${VER_NODEJS_RE}([.-]|$)") @@ -195,7 +195,7 @@ setup_node_pnpm() { local VER_PNPM_REQ="${1:-}" UNAME ARCH URL_PNPM TMPDIR ; UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && ARCH=$(uname -m | sed -e 's/x86_64/x64/' -e 's/aarch64/arm64/' -e 's/armv7l/arm/') \ - && local VERS_PNPM=$(curl -fsSL "https://api.github.com/repos/pnpm/pnpm/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -viE '(alpha|beta|rc)' | grep -Po '\d[\d.]+' | sort -rV) \ + && local VERS_PNPM=$(curl_fsSL "https://api.github.com/repos/pnpm/pnpm/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -viE '(alpha|beta|rc)' | grep -Po '\d[\d.]+' | sort -rV) \ && if [ -n "${VER_PNPM_REQ}" ]; then local VER_PNPM_RE=${VER_PNPM_REQ#v} \ && VER_PNPM_RE=$(printf '%s' "$VER_PNPM_RE" | sed 's/[.]/\\./g') \ @@ -208,13 +208,13 @@ setup_node_pnpm() { && if [ "${VER_PNPM%%.*}" -ge 11 ]; then URL_PNPM="https://github.com/pnpm/pnpm/releases/download/v${VER_PNPM}/pnpm-${UNAME}-${ARCH}.tar.gz" \ && echo "@ Installing pnpm v${VER_PNPM} from: ${URL_PNPM}" \ - && TMPDIR=$(mktemp -d) && curl -fL "${URL_PNPM}" | tar -xzf - -C "${TMPDIR}" \ + && TMPDIR=$(mktemp -d) && curl_fsSL "${URL_PNPM}" | tar -xzf - -C "${TMPDIR}" \ && sudo mv -f "${TMPDIR}/pnpm" /opt/node/bin/pnpm \ && { [ -d "${TMPDIR}/dist" ] && sudo rm -rf /opt/node/bin/dist && sudo mv "${TMPDIR}/dist" /opt/node/bin/ ; rm -rf "${TMPDIR}" ; true ; } ; else URL_PNPM="https://github.com/pnpm/pnpm/releases/download/v${VER_PNPM}/pnpm-${UNAME}-${ARCH}" \ && echo "@ Installing pnpm v${VER_PNPM} from: ${URL_PNPM}" \ - && sudo curl -fL "${URL_PNPM}" -o /opt/node/bin/pnpm ; + && sudo curl_fsSL -o /opt/node/bin/pnpm "${URL_PNPM}" ; fi \ && sudo chmod +x /opt/node/bin/pnpm && sudo ln -sf /opt/node/bin/pnpm /usr/local/bin/pnpm \ && echo 'export PNPM_STORE_DIR=/opt/node/pnpm-store' | sudo tee /etc/profile.d/path-pnpm.sh >/dev/null ; @@ -225,7 +225,7 @@ setup_node_bun() { local VER_BUN_REQ="${1:-}" \ && local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && local ARCH=$(uname -m | sed -e 's/x86_64/x64/' ) \ - && local VERS_BUN=$(curl -sL "https://api.github.com/repos/oven-sh/bun/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po 'bun-v\K\d+\.\d+\.\d+' | sort -rV) \ + && local VERS_BUN=$(curl_fsSL "https://api.github.com/repos/oven-sh/bun/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po 'bun-v\K\d+\.\d+\.\d+' | sort -rV) \ && if [ -n "${VER_BUN_REQ}" ]; then local VER_BUN_RE=${VER_BUN_REQ#v} \ && VER_BUN_RE=$(printf '%s' "$VER_BUN_RE" | sed 's/[.]/\\./g') \ @@ -249,8 +249,8 @@ setup_GO() { local VER_GO_REQ="${1:-}" \ && local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && local ARCH=$(dpkg --print-architecture) \ - && local VERS_GO=$(curl -sL "https://api.github.com/repos/golang/go/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -v 'rc' | grep -Po '\d[\d.]+' | sort -rV) \ - && { [ -n "${VERS_GO}" ] || VERS_GO=$(curl -sL "https://go.dev/dl/?mode=json" | grep -Po '(?<="version": "go)\d[\d.]+' | sort -rV) ; } \ + && local VERS_GO=$(curl_fsSL "https://api.github.com/repos/golang/go/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -v 'rc' | grep -Po '\d[\d.]+' | sort -rV) \ + && { [ -n "${VERS_GO}" ] || VERS_GO=$(curl_fsSL "https://go.dev/dl/?mode=json" | grep -Po '(?<="version": "go)\d[\d.]+' | sort -rV) ; } \ && if [ -n "${VER_GO_REQ}" ]; then local VER_GO_RE=${VER_GO_REQ#v} \ && VER_GO_RE=$(printf '%s' "$VER_GO_RE" | sed 's/[.]/\\./g') \ @@ -273,7 +273,7 @@ setup_GO() { setup_rust() { - curl -sSf https://sh.rustup.rs | sudo sh -c ' + curl_fsSL https://sh.rustup.rs | sudo sh -c ' export CARGO_HOME=/opt/cargo && export RUSTUP_HOME=/opt/rust sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable ' @@ -288,7 +288,7 @@ setup_rust() { setup_R_base() { local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ - && curl -sL https://cloud.r-project.org/bin/${UNAME}/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc \ + && curl_fsSL https://cloud.r-project.org/bin/${UNAME}/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc \ && echo "deb https://cloud.r-project.org/bin/${UNAME}/ubuntu $(lsb_release -cs)-cran40/" | sudo tee /etc/apt/sources.list.d/cran.list >/dev/null \ && install_apt /opt/utils/install_list_R_base.apt \ && echo "options(repos=structure(c(CRAN=\"https://cloud.r-project.org\")))" | sudo tee -a /etc/R/Rprofile.site \ @@ -304,7 +304,7 @@ setup_julia() { && local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && local ARCH_1=$(uname -m) \ && local ARCH_2=$(uname -m | sed -e 's/x86_64/x64/') \ - && local VERS_JULIA=$(curl -sL "https://api.github.com/repos/JuliaLang/julia/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -P -v '(rc|alpha|beta)' | grep -Po '\d[\d.]+' | sort -rV) \ + && local VERS_JULIA=$(curl_fsSL "https://api.github.com/repos/JuliaLang/julia/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -P -v '(rc|alpha|beta)' | grep -Po '\d[\d.]+' | sort -rV) \ && if [ -n "${VER_JULIA_REQ}" ]; then local VER_JULIA_RE=${VER_JULIA_REQ#v} \ && VER_JULIA_RE=$(printf '%s' "$VER_JULIA_RE" | sed 's/[.]/\\./g') \ @@ -329,7 +329,7 @@ setup_julia() { setup_lua_base() { local VER_LUA_REQ="${1:-}" && VER_LUA_REQ="${VER_LUA_REQ#v}" \ && local LUA_HOME="${LUA_HOME:-/opt/lua}" \ - && local VERS_JSON=$(curl -fsSL "https://api.github.com/repos/lua/lua/releases?per_page=50") \ + && local VERS_JSON=$(curl_fsSL "https://api.github.com/repos/lua/lua/releases?per_page=50") \ && local VERS_LUA=$(printf '%s\n' "$VERS_JSON" | sed -n 's/.*"tag_name"[[:space:]]*:[[:space:]]*"v\([0-9][0-9.]*\)".*/\1/p') \ && if [ -n "$VER_LUA_REQ" ]; then local VER_LUA=$(printf '%s\n' "$VERS_LUA" | grep -F -x "$VER_LUA_REQ" | sed -n '1p') @@ -351,7 +351,7 @@ setup_lua_rocks() { ## https://github.com/luarocks/luarocks/wiki/Installation-instructions-for-Unix local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && local VER_LUA_ROCKS_REQ="${1:-}" \ - && local VERS_LUA_ROCKS=$(curl -sL https://luarocks.github.io/luarocks/releases/ | grep "${UNAME}" | grep -viE '(alpha|beta|rc)' | grep -Po '\d+\.\d+(\.\d+)?' | sort -rV) \ + && local VERS_LUA_ROCKS=$(curl_fsSL "https://luarocks.github.io/luarocks/releases/" | grep "${UNAME}" | grep -viE '(alpha|beta|rc)' | grep -Po '\d+\.\d+(\.\d+)?' | sort -rV) \ && if [ -n "${VER_LUA_ROCKS_REQ}" ]; then local VER_LUA_ROCKS_RE=${VER_LUA_ROCKS_REQ#v} \ && VER_LUA_ROCKS_RE=$(printf '%s' "$VER_LUA_ROCKS_RE" | sed 's/[.]/\\./g') \ @@ -375,7 +375,7 @@ setup_bazel() { local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && local ARCH=$(uname -m | sed -e 's/aarch64/arm64/') \ && local VER_BAZEL_REQ="${1:-}" \ - && local VERS_BAZEL=$(curl -sL "https://api.github.com/repos/bazelbuild/bazel/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -vi 'rc' | grep -Po '\d[\d.]+' | sort -rV) \ + && local VERS_BAZEL=$(curl_fsSL "https://api.github.com/repos/bazelbuild/bazel/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -vi 'rc' | grep -Po '\d[\d.]+' | sort -rV) \ && if [ -n "${VER_BAZEL_REQ}" ]; then local VER_BAZEL_RE=${VER_BAZEL_REQ#v} \ && VER_BAZEL_RE=$(printf '%s' "$VER_BAZEL_RE" | sed 's/[.]/\\./g') \ @@ -385,7 +385,7 @@ setup_bazel() { fi \ && [ -n "${VER_BAZEL}" ] \ && local URL_BAZEL="https://github.com/bazelbuild/bazel/releases/download/${VER_BAZEL}/bazel-${VER_BAZEL}-installer-${UNAME}-${ARCH}.sh" \ - && curl -o /tmp/bazel.sh -sL "${URL_BAZEL}" && chmod +x /tmp/bazel.sh \ + && curl_fsSL -o /tmp/bazel.sh "${URL_BAZEL}" && chmod +x /tmp/bazel.sh \ && sudo /tmp/bazel.sh && rm /tmp/bazel.sh ; type bazel && echo "@ Version of bazel: $(bazel --version)" || return 1 ; } @@ -393,7 +393,7 @@ setup_bazel() { setup_gradle() { local VER_GRADLE_REQ="${1:-}" \ - && local VERS_GRADLE=$(curl -sL "https://api.github.com/repos/gradle/gradle/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -v 'M' | grep -vi 'rc' | grep -Po '\d[\d.]+' | sort -rV) \ + && local VERS_GRADLE=$(curl_fsSL "https://api.github.com/repos/gradle/gradle/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -v 'M' | grep -vi 'rc' | grep -Po '\d[\d.]+' | sort -rV) \ && if [ -n "${VER_GRADLE_REQ}" ]; then local VER_GRADLE_RE=${VER_GRADLE_REQ#v} \ && VER_GRADLE_RE=$(printf '%s' "$VER_GRADLE_RE" | sed 's/[.]/\\./g') \ @@ -413,7 +413,7 @@ setup_yq() { local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/' -e 's/armv7l/arm/') ; case "$ARCH" in amd64|arm64|arm) ;; *) echo "Unsupported architecture for yq: $(uname -m)"; return 1 ;; esac local VER_YQ="" VER_YQ_REQ="${1:-}" \ - && local VERS_YQ=$(curl -sL "https://api.github.com/repos/mikefarah/yq/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po 'v\K[\d.]+' | sort -rV) \ + && local VERS_YQ=$(curl_fsSL "https://api.github.com/repos/mikefarah/yq/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po 'v\K[\d.]+' | sort -rV) \ && if [ -n "${VER_YQ_REQ}" ]; then local VER_YQ_RE=${VER_YQ_REQ#v} && VER_YQ_RE=$(printf '%s' "$VER_YQ_RE" | sed 's/[.]/\\./g') \ && VER_YQ=$(echo "${VERS_YQ}" | grep -m1 -E "^${VER_YQ_RE}([.-]|$)") @@ -423,7 +423,7 @@ setup_yq() { && [ -n "${VER_YQ}" ] \ && local URL_YQ="https://github.com/mikefarah/yq/releases/download/v${VER_YQ}/yq_linux_${ARCH}" \ && echo "Installing yq v${VER_YQ} for arch ${ARCH} from: ${URL_YQ}" \ - && curl -fSL "${URL_YQ}" -o /tmp/yq \ + && curl_fsSL -o /tmp/yq "${URL_YQ}" \ && sudo install -m 0755 -D /tmp/yq /opt/bin/yq \ && sudo ln -sf /opt/bin/yq /usr/bin/yq \ && rm -f /tmp/yq ; diff --git a/docker_atom/work/script-setup-db-clients.sh b/docker_atom/work/script-setup-db-clients.sh index 189b8ecd..dcfce3b6 100755 --- a/docker_atom/work/script-setup-db-clients.sh +++ b/docker_atom/work/script-setup-db-clients.sh @@ -4,7 +4,7 @@ setup_postgresql_client() { local VER_PG=${1:-"18"} # from: https://www.postgresql.org/download/linux/ubuntu/ - curl "https://www.postgresql.org/media/keys/ACCC4CF8.asc" | sudo tee /etc/apt/trusted.gpg.d/postgresql.asc + curl_fsSL "https://www.postgresql.org/media/keys/ACCC4CF8.asc" | sudo tee /etc/apt/trusted.gpg.d/postgresql.asc echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list # will download ~9MB files and use ~55MB disk after installation sudo apt-get update && sudo apt-get -y install "postgresql-client-${VER_PG}" @@ -14,7 +14,7 @@ setup_postgresql_client() { setup_duckdb() { local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ && local VER_DUCKDB_REQ="${1:-}" \ - && local VERS_DUCKDB=$(curl -s https://api.github.com/repos/duckdb/duckdb/releases | grep -oP '"tag_name":\s*"\K[^"]+' | sed 's/^v//' | sort -rV) \ + && local VERS_DUCKDB=$(curl_fsSL https://api.github.com/repos/duckdb/duckdb/releases | grep -oP '"tag_name":\s*"\K[^"]+' | sed 's/^v//' | sort -rV) \ && if [ -n "${VER_DUCKDB_REQ}" ]; then local VER_DUCKDB_RE=${VER_DUCKDB_REQ#v} \ && VER_DUCKDB_RE=$(printf '%s' "$VER_DUCKDB_RE" | sed 's/[.]/\\./g') \ @@ -29,7 +29,7 @@ setup_duckdb() { && local URL_DUCKDB="https://github.com/duckdb/duckdb/releases/download/v${VER_DUCKDB}/${FILE_DUCKDB}" \ && local TMP_FILE="/tmp/duckdb-${VER_DUCKDB}.${EXT_DUCKDB}" \ && echo "Downloading DuckDB ${VER_DUCKDB} from: ${URL_DUCKDB}" \ - && curl -L "${URL_DUCKDB}" -o "${TMP_FILE}" \ + && curl_fsSL -o "${TMP_FILE}" "${URL_DUCKDB}" \ && if [ "${FILE_DUCKDB##*.}" = zip ]; then unzip -q "${TMP_FILE}" -d /tmp; else tar -xzf "${TMP_FILE}" -C /tmp; fi \ && sudo mkdir -pv /opt/bin \ && sudo install -m 0755 /tmp/duckdb /opt/bin/duckdb \ @@ -48,7 +48,7 @@ setup_mongosh_client() { local VER_MONGOSH=${1:-"8.0"} # from: https://www.mongodb.com/docs/mongodb-shell/install/ local DISTRO=$(lsb_release -cs) - curl -sL https://www.mongodb.org/static/pgp/server-${VER_MONGOSH}.asc | sudo tee /etc/apt/trusted.gpg.d/mongodb.asc + curl_fsSL "https://www.mongodb.org/static/pgp/server-${VER_MONGOSH}.asc" | sudo tee /etc/apt/trusted.gpg.d/mongodb.asc echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu ${DISTRO}/mongodb-org/${VER_MONGOSH} multiverse" | sudo tee "/etc/apt/sources.list.d/mongodb-org-${VER_MONGOSH}.list" >/dev/null # will download ~38MB files and use ~218MB disk after installation sudo apt-get update && sudo apt-get -y install mongodb-mongosh @@ -58,7 +58,7 @@ setup_mongosh_client() { setup_redis_client() { # from https://redis.io/docs/getting-started/installation/install-redis-on-linux/ - curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg + curl_fsSL "https://packages.redis.io/gpg" | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list sudo apt-get update && sudo apt-get -y install redis-tools type redis-cli && echo "@ Version of redis-cli: $(redis-cli --version)" || return 1 @@ -80,7 +80,7 @@ setup_oracle_client() { PAGE="https://www.oracle.com/database/technologies/instant-client/linux-${OARCH}-downloads.html" echo "Finding download URL for Oracle client ${PKG_ARCH}: ${PAGE}" - DL=$(curl -s "$PAGE" \ + DL=$(curl_fsSL "$PAGE" \ | grep -oP "//download.oracle.com/otn_software/linux/instantclient/[0-9]+/instantclient-basic-linux\.${PKG_ARCH}-[0-9.]+\.zip" \ | head -n1) @@ -93,7 +93,7 @@ setup_oracle_client() { local ZIP="instantclient-basic-linux.${PKG_ARCH}-${VER}.zip" local URL="https://download.oracle.com/otn_software/linux/instantclient/${BUILD}/${ZIP}" echo "@ Installing Oracle Instant Client ${VER} (${PKG_ARCH} build ${BUILD}): ${URL}" - curl -L -o "/tmp/${ZIP}" "$URL" + curl_fsSL -L -o "/tmp/${ZIP}" "$URL" sudo mkdir -pv /opt/oracle sudo unzip -o "/tmp/${ZIP}" "instantclient_*/*" -d /opt/oracle diff --git a/docker_atom/work/script-setup-net.sh b/docker_atom/work/script-setup-net.sh index 9c0f0fe6..a30cf0c3 100755 --- a/docker_atom/work/script-setup-net.sh +++ b/docker_atom/work/script-setup-net.sh @@ -1,7 +1,9 @@ +. /opt/utils/script-utils.sh + setup_traefik() { local ARCH=$(dpkg --print-architecture) \ && local VER_TRAEFIK_REQ="${1:-}" \ - && local VERS_TRAEFIK=$(curl -sL "https://api.github.com/repos/traefik/traefik/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -vi 'rc' | grep -Po '\d[\d.]+' | sort -rV) \ + && local VERS_TRAEFIK=$(curl_fsSL "https://api.github.com/repos/traefik/traefik/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -vi 'rc' | grep -Po '\d[\d.]+' | sort -rV) \ && if [ -n "${VER_TRAEFIK_REQ}" ]; then local VER_TRAEFIK_RE=${VER_TRAEFIK_REQ#v} \ && VER_TRAEFIK_RE=$(printf '%s' "$VER_TRAEFIK_RE" | sed 's/[.]/\\./g') \ @@ -11,7 +13,7 @@ setup_traefik() { fi \ && [ -n "${VER_TRAEFIK}" ] \ && local URL_TRAEFIK="https://github.com/traefik/traefik/releases/download/v${VER_TRAEFIK}/traefik_v${VER_TRAEFIK}_linux_${ARCH}.tar.gz" \ - && curl -o /tmp/TMP.tgz -sL "${URL_TRAEFIK}" \ + && curl_fsSL -o /tmp/TMP.tgz "${URL_TRAEFIK}" \ && sudo mkdir -pv /opt/bin && sudo tar -C /opt/bin -xzf /tmp/TMP.tgz traefik && rm /tmp/TMP.tgz \ && sudo chmod +x /opt/bin/traefik && sudo ln -sf /opt/bin/traefik /usr/bin/ ; type traefik && echo "@ Version of traefik: $(traefik version)" || return 1 ; @@ -21,7 +23,7 @@ setup_caddy() { local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ && local VER_CADDY_REQ="${1:-}" \ - && local VERS_CADDY=$(curl -sL "https://api.github.com/repos/caddyserver/caddy/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -viE '(beta|rc)' | grep -Po '(\d[\d|.]+)' | sort -rV) \ + && local VERS_CADDY=$(curl_fsSL "https://api.github.com/repos/caddyserver/caddy/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -viE '(beta|rc)' | grep -Po '(\d[\d|.]+)' | sort -rV) \ && if [ -n "${VER_CADDY_REQ}" ]; then local VER_CADDY_RE=${VER_CADDY_REQ#v} \ && VER_CADDY_RE=$(printf '%s' "$VER_CADDY_RE" | sed 's/[.]/\\./g') \ @@ -32,7 +34,7 @@ setup_caddy() { && [ -n "${VER_CADDY}" ] \ && local URL_CADDY="https://github.com/caddyserver/caddy/releases/download/v${VER_CADDY}/caddy_${VER_CADDY}_${UNAME}_${ARCH}.tar.gz" \ && echo "Downloading Caddy ${VER_CADDY} from ${URL_CADDY}" \ - && curl -o /tmp/TMP.tgz -sL "${URL_CADDY}" && tar -C /tmp/ -xzf /tmp/TMP.tgz && rm /tmp/TMP.tgz \ + && curl_fsSL -o /tmp/TMP.tgz "${URL_CADDY}" && tar -C /tmp/ -xzf /tmp/TMP.tgz && rm /tmp/TMP.tgz \ && sudo mkdir -pv /opt/bin/ && sudo mv /tmp/caddy /opt/bin/ && sudo ln -sf /opt/bin/caddy /usr/local/bin/ ; type caddy && echo "@ Version of caddy: $(caddy version)" || return 1 ; } @@ -42,7 +44,7 @@ setup_oauth2_proxy() { case "$ARCH" in amd64|arm64|armv7) ;; *) echo "Unsupported architecture for oauth2-proxy: $(uname -m)"; return 1 ;; esac local VER_OAUTH2_PROXY_REQ="${1:-}" \ - && local VERS_OAUTH2_PROXY=$(curl -sL "https://api.github.com/repos/oauth2-proxy/oauth2-proxy/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po '(?<=v)?\d[\d.]+' | sort -rV) \ + && local VERS_OAUTH2_PROXY=$(curl_fsSL "https://api.github.com/repos/oauth2-proxy/oauth2-proxy/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po '(?<=v)?\d[\d.]+' | sort -rV) \ && if [ -n "${VER_OAUTH2_PROXY_REQ}" ]; then local VER_OAUTH2_PROXY_RE=${VER_OAUTH2_PROXY_REQ#v} \ && VER_OAUTH2_PROXY_RE=$(printf '%s' "$VER_OAUTH2_PROXY_RE" | sed 's/[.]/\\./g') \ @@ -54,7 +56,7 @@ setup_oauth2_proxy() { && local FILE_OAUTH2_PROXY="oauth2-proxy-v${VER_OAUTH2_PROXY}.linux-${ARCH}.tar.gz" \ && local URL_OAUTH2_PROXY="https://github.com/oauth2-proxy/oauth2-proxy/releases/download/v${VER_OAUTH2_PROXY}/${FILE_OAUTH2_PROXY}" \ && echo "Installing oauth2-proxy v${VER_OAUTH2_PROXY} for arch ${ARCH} from: ${URL_OAUTH2_PROXY}" \ - && curl -fSL "${URL_OAUTH2_PROXY}" -o /tmp/oauth2-proxy.tar.gz \ + && curl_fsSL -o /tmp/oauth2-proxy.tar.gz "${URL_OAUTH2_PROXY}" \ && tar -xzf /tmp/oauth2-proxy.tar.gz -C /tmp \ && sudo install -m 0755 -D "/tmp/oauth2-proxy-v${VER_OAUTH2_PROXY}.linux-${ARCH}/oauth2-proxy" /opt/bin/oauth2-proxy \ && sudo ln -sf /opt/bin/oauth2-proxy /usr/bin/oauth2-proxy \ diff --git a/docker_atom/work/script-setup-sys.sh b/docker_atom/work/script-setup-sys.sh index 2a40b0cc..efccd51c 100755 --- a/docker_atom/work/script-setup-sys.sh +++ b/docker_atom/work/script-setup-sys.sh @@ -4,7 +4,7 @@ setup_tini() { local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ && local VER_TINI_REQ="${1:-}" \ - && local VERS_TINI=$(curl -sL "https://api.github.com/repos/krallin/tini/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po '\d[\d.]+' | sort -rV) \ + && local VERS_TINI=$(curl_fsSL "https://api.github.com/repos/krallin/tini/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po '\d[\d.]+' | sort -rV) \ && if [ -n "${VER_TINI_REQ}" ]; then local VER_TINI_RE=${VER_TINI_REQ#v} \ && VER_TINI_RE=$(printf '%s' "$VER_TINI_RE" | sed 's/[.]/\\./g') \ @@ -15,7 +15,7 @@ setup_tini() { && [ -n "${VER_TINI}" ] \ && local URL_TINI="https://github.com/krallin/tini/releases/download/v${VER_TINI}/tini-${ARCH}" \ && echo "Downloading Tini ${VER_TINI} from ${URL_TINI}" \ - && sudo curl -o /usr/bin/tini -sL "$URL_TINI" && sudo chmod +x /usr/bin/tini ; + && sudo curl_fsSL -o /usr/bin/tini "$URL_TINI" && sudo chmod +x /usr/bin/tini ; type tini && echo "@ Version of tini: $(tini --version)" || return 1 ; # ref: https://cloud-atlas.readthedocs.io/zh-cn/latest/docker/init/docker_tini.html # to run multi-proces with tini: use a bash script ends with the following code @@ -27,7 +27,7 @@ setup_supervisord() { local UNAME=$(uname | tr '[:upper:]' '[:lower:]') \ && local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ && local VER_SUPERVISORD_REQ="${1:-}" \ - && local VERS_SUPERVISORD=$(curl -sL "https://api.github.com/repos/LabNow-ai/supervisord/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po '(\d[\d|.]+)' | sort -rV) \ + && local VERS_SUPERVISORD=$(curl_fsSL "https://api.github.com/repos/LabNow-ai/supervisord/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | grep -Po '(\d[\d|.]+)' | sort -rV) \ && if [ -n "${VER_SUPERVISORD_REQ}" ]; then local VER_SUPERVISORD_RE=${VER_SUPERVISORD_REQ#v} \ && VER_SUPERVISORD_RE=$(printf '%s' "$VER_SUPERVISORD_RE" | sed 's/[.]/\\./g') \ @@ -38,7 +38,7 @@ setup_supervisord() { && [ -n "${VER_SUPERVISORD}" ] \ && local URL_SUPERVISORD="https://github.com/LabNow-ai/supervisord/releases/download/v${VER_SUPERVISORD}/supervisord_${VER_SUPERVISORD}_${UNAME}_${ARCH}.tar.gz" \ && echo "Downloading Supervisord ${VER_SUPERVISORD} from ${URL_SUPERVISORD}" \ - && curl -o /tmp/TMP.tgz -sL $URL_SUPERVISORD && tar -C /tmp/ -xzf /tmp/TMP.tgz && rm /tmp/TMP.tgz \ + && curl_fsSL -o /tmp/TMP.tgz "$URL_SUPERVISORD" && tar -C /tmp/ -xzf /tmp/TMP.tgz && rm /tmp/TMP.tgz \ && sudo mkdir -pv /opt/bin/ && sudo mv /tmp/supervisord /opt/bin/ && sudo ln -sf /opt/bin/supervisord /usr/local/bin/ ; type supervisord && echo "@ Version of supervisord: $(supervisord version)" || return 1 ; } diff --git a/docker_atom/work/script-utils.sh b/docker_atom/work/script-utils.sh index 32ef1b25..3732cf15 100755 --- a/docker_atom/work/script-utils.sh +++ b/docker_atom/work/script-utils.sh @@ -26,16 +26,16 @@ install_julia() { julia -e "import Pkg; l=filter(x->length(x)>0, [split(i,r\"% install_octave() { cut -d "%" -f 1 "$1" | sed '/^$/d' | xargs -r -n1 -I {} sh -c 'octave --eval "pkg install -forge {}"' ; } # function to download a ZIP file and unzip it to /opt/ -install_zip() { curl -o /tmp/TMP.zip -sL "$1" && sudo unzip -q -d /opt/ /tmp/TMP.zip && rm /tmp/TMP.zip ; } +install_zip() { curl -fsSL -o /tmp/TMP.zip "$1" && sudo unzip -q -d /opt/ /tmp/TMP.zip && rm /tmp/TMP.zip ; } # function to download a .tar.gz file and unzip it to /opt/, add a second argument to extract only those file -install_tar_gz() { curl -o /tmp/TMP.tgz -sL "$1" && sudo tar -C /opt/ -xzf /tmp/TMP.tgz ${2:-} && rm /tmp/TMP.tgz ; } +install_tar_gz() { curl -fsSL -o /tmp/TMP.tgz "$1" && sudo tar -C /opt/ -xzf /tmp/TMP.tgz ${2:-} && rm /tmp/TMP.tgz ; } # function to download a .tar.bz file and unzip it to /opt/, add a second argument to extract only those file -install_tar_bz() { curl -o /tmp/TMP.tbz -sL "$1" && sudo tar -C /opt/ -xjf /tmp/TMP.tbz ${2:-} && rm /tmp/TMP.tbz ; } +install_tar_bz() { curl -fsSL -o /tmp/TMP.tbz "$1" && sudo tar -C /opt/ -xjf /tmp/TMP.tbz ${2:-} && rm /tmp/TMP.tbz ; } # function to download a .tar.xz file and unzip it to /opt/, add a second argument to extract only those file -install_tar_xz() { curl -o /tmp/TMP.txz -sL "$1" && sudo tar -C /opt/ -xJf /tmp/TMP.txz ${2:-} && rm /tmp/TMP.txz ; } +install_tar_xz() { curl -fsSL -o /tmp/TMP.txz "$1" && sudo tar -C /opt/ -xJf /tmp/TMP.txz ${2:-} && rm /tmp/TMP.txz ; } # function to install java packages from a text file which lists JAR file maven full names (add comments with % char) install_mvn() { cut -d "%" -f 1 "$1" | xargs -r -n1 -I {} mvn dependency:copy -DlocalRepositoryDirectory="/tmp/m2repo" -Djavax.net.ssl.trustStorePassword=changeit -Dartifact="{}" -DoutputDirectory="${2:-}" ; } @@ -64,8 +64,8 @@ list_installed_packages() { type conda && echo "@ Version of Conda and packages:" && conda info && conda list | grep -v "" type mamba && echo "@ Version of Mamba and packages:" && mamba info && mamba list | grep -v "" type node && echo "@ Version of NodeJS and packages:" && node --version && npm --version && npm list -g --depth 0 - type java && echo "@ Version of Java (JRE):" && java -version - type javac && echo "@ Version of Java (JDK):" && javac -version + type java && echo "@ Version of Java (JRE):" && java -version + type javac && echo "@ Version of Java (JDK):" && javac -version type R && echo "@ Version of R and libraries:" && R --version && R -e "R.Version()\$version.string;installed.packages()[,c(3,10)]" type julia && echo "@ Version of Julia and packages" && julia --version && julia -e "using Pkg; for(k,v) in Pkg.dependencies(); println(v.name,\"==\",v.version); end" type go && echo "@ Version of golang and packages:" && go version && go list ... @@ -92,3 +92,5 @@ fix_permission() { -exec chmod +6000 -- {} \+; done } + +curl_fsSL() { curl --fail --silent --show-error --location --retry 3 --retry-delay 2 "$@" ; } diff --git a/docker_cuda/nvidia-cuda.Dockerfile b/docker_cuda/nvidia-cuda.Dockerfile index 06734f52..429b1364 100644 --- a/docker_cuda/nvidia-cuda.Dockerfile +++ b/docker_cuda/nvidia-cuda.Dockerfile @@ -14,7 +14,7 @@ ENV NVIDIA_DISABLE_REQUIRE=1 RUN set -eux && echo ${CUDA_VERSION} && nvcc --version \ # HACK & FIX: for some old version of NVIDIA docker images, sys python version is too old, update debpython && URL_PY3_DEP="https://salsa.debian.org/cpython-team/python3-defaults/-/archive/master/python3-defaults-master.zip?path=debpython" \ - && curl -o /tmp/TMP.zip -sL "${URL_PY3_DEP}" && unzip -q -d /tmp/ /tmp/TMP.zip && rm /tmp/TMP.zip \ + && curl -fsSL -o /tmp/TMP.zip "${URL_PY3_DEP}" && unzip -q -d /tmp/ /tmp/TMP.zip && rm /tmp/TMP.zip \ && mkdir -pv /usr/share/python3/debpython/ \ && cp -rf /tmp/python3-defaults-master-debpython/debpython/* /usr/share/python3/debpython/ \ && rm -rf /tmp/python3-defaults-master-debpython \ diff --git a/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext-mirror.sh b/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext-mirror.sh index 84587de8..0ac7ad8c 100644 --- a/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext-mirror.sh +++ b/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext-mirror.sh @@ -13,8 +13,8 @@ add_apt_source() { local keyring_path="${APT_KEYRING_DIR}/${keyring_name}" local list_path="${APT_SOURCE_DIR}/${list_name}" - curl -fsSL "$key_url" | gpg --dearmor | sudo tee "$keyring_path" >/dev/null - curl -fsSL "$list_url" | sudo tee "$list_path" >/dev/null + curl_fsSL "$key_url" | gpg --dearmor | sudo tee "$keyring_path" >/dev/null + curl_fsSL "$list_url" | sudo tee "$list_path" >/dev/null } diff --git a/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext.sh b/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext.sh index 236ad5dd..b11f620e 100644 --- a/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext.sh +++ b/docker_db_postgres/rootfs/opt/utils/script-setup-pg-ext.sh @@ -1,4 +1,4 @@ -if [ ${PG_MAJOR} != "17" ]; then +if [ "${PG_MAJOR}" -lt 17 ]; then # Fix pgagent for PG 15 / 16 sudo rm -rf /usr/share/postgresql/${PG_MAJOR}/extension/pgagent* sudo mv /usr/share/postgresql/17/extension/pgagent* /usr/share/postgresql/${PG_MAJOR}/extension/ @@ -18,11 +18,11 @@ setup_pg_search() { ## ref1: https://docs.paradedb.com/deploy/self-hosted/extension ## ref2: https://github.com/paradedb/paradedb local ARCH="amd64" \ - && local VER_PG_SEARCH=$(curl -fsSL "https://api.github.com/repos/paradedb/paradedb/releases?per_page=50" | grep -Po '(?<="tag_name": "v)[^"]+' | grep -v -- '-rc' | head -1) \ + && local VER_PG_SEARCH=$(curl_fsSL "https://api.github.com/repos/paradedb/paradedb/releases?per_page=50" | grep -Po '(?<="tag_name": "v)[^"]+' | grep -v -- '-rc' | head -1) \ && local URL_PG_SEARCH="https://github.com/paradedb/paradedb/releases/download/v${VER_PG_SEARCH}/postgresql-${PG_MAJOR}-pg-search_${VER_PG_SEARCH}-1PARADEDB-$(lsb_release -cs)_amd64.deb" \ && echo "Downloading pg_search ${VER_PG_SEARCH} from: ${URL_PG_SEARCH}" \ && mkdir -pv /tmp/pg_search/ && cd /tmp/pg_search \ - && curl -fsSL -O "${URL_PG_SEARCH}" \ + && curl_fsSL -O "${URL_PG_SEARCH}" \ && sudo apt-get -qq install -yq --no-install-recommends libopenblas0 \ && sudo dpkg -i *.deb } @@ -39,7 +39,7 @@ setup_apache_age() { setup_pgvectorscale() { ## ref: https://github.com/timescale/pgvectorscale local ARCH="amd64" \ - && local VER_PGVS=$(curl -sL "https://api.github.com/repos/timescale/pgvectorscale/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[\d.]+' ) \ + && local VER_PGVS=$(curl_fsSL "https://api.github.com/repos/timescale/pgvectorscale/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[\d.]+' ) \ && local URL_PGVS="https://github.com/timescale/pgvectorscale/releases/download/${VER_PGVS}/pgvectorscale-${VER_PGVS}-pg${PG_MAJOR}-${ARCH}.zip" \ && mkdir -pv /tmp/pgvectorscale/ && cd /tmp/pgvectorscale \ && install_zip "${URL_PGVS}" && sudo mv /opt/pgvectorscal* /tmp/pgvectorscale/ \ @@ -50,10 +50,10 @@ setup_pgroonga(){ ## ref1: https://pgroonga.github.io/tutorial/ ## ref2: https://github.com/pgroonga/docker mkdir -pv /tmp/pgroonga && cd /tmp/pgroonga - curl -fsSL -O https://apache.jfrog.io/artifactory/arrow/debian/apache-arrow-apt-source-latest-$(lsb_release -cs).deb \ + curl_fsSL -O https://apache.jfrog.io/artifactory/arrow/debian/apache-arrow-apt-source-latest-$(lsb_release -cs).deb \ && sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release -cs).deb \ && rm apache-arrow-apt-source-latest-$(lsb_release -cs).deb \ - && curl -fsSL -O https://packages.groonga.org/debian/groonga-apt-source-latest-$(lsb_release -cs).deb \ + && curl_fsSL -O https://packages.groonga.org/debian/groonga-apt-source-latest-$(lsb_release -cs).deb \ && sudo apt install -y -V ./groonga-apt-source-latest-$(lsb_release -cs).deb \ && rm groonga-apt-source-latest-$(lsb_release -cs).deb \ && sudo apt update && sudo apt install -y -V \ diff --git a/docker_docker_kit/work/script-setup-docker.sh b/docker_docker_kit/work/script-setup-docker.sh index 96cee65c..7adbf80d 100755 --- a/docker_docker_kit/work/script-setup-docker.sh +++ b/docker_docker_kit/work/script-setup-docker.sh @@ -3,20 +3,20 @@ setup_docker_compose() { local ARCH=$(uname -m) \ - && local VER_COMPOSE=$(curl -sL "https://api.github.com/repos/docker/compose/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[.\d]+') \ + && local VER_COMPOSE=$(curl_fsSL "https://api.github.com/repos/docker/compose/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[.\d]+') \ && local URL_COMPOSE="https://github.com/docker/compose/releases/download/v${VER_COMPOSE}/docker-compose-linux-${ARCH}" \ && echo "Downloading Compose from: ${URL_COMPOSE}" \ - && sudo curl -o /usr/bin/docker-compose -sL ${URL_COMPOSE} \ + && sudo curl_fsSL -o /usr/bin/docker-compose "${URL_COMPOSE}" \ && sudo chmod +x /usr/bin/docker-compose ; type docker-compose && echo "@ Version of docker-compose: $(docker-compose --version)" || return 1 ; } setup_docker_syncer() { local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ - && local VER_SYNCER="$(curl -sL "https://api.github.com/repos/AliyunContainerService/image-syncer/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[.\d]+')" \ + && local VER_SYNCER="$(curl_fsSL "https://api.github.com/repos/AliyunContainerService/image-syncer/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[.\d]+')" \ && local URL_SYNCER="https://github.com/AliyunContainerService/image-syncer/releases/download/v${VER_SYNCER}/image-syncer-v${VER_SYNCER}-linux-${ARCH}.tar.gz" \ && echo "Downloading image-syncer from: ${URL_SYNCER}" \ - && curl -o /tmp/image_syncer.tgz -sL ${URL_SYNCER} \ + && curl_fsSL -o /tmp/image_syncer.tgz "${URL_SYNCER}" \ && mkdir -pv /tmp/image_syncer && tar -zxvf /tmp/image_syncer.tgz -C /tmp/image_syncer \ && sudo chmod +x /tmp/image_syncer/image-syncer \ && sudo mv /tmp/image_syncer/image-syncer /usr/bin/ \ From 0d662609ea2aba821c85b5a8ef9bd32c252f4381 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Sun, 15 Nov 2026 07:35:26 +0000 Subject: [PATCH 7/8] fix sudo curl --- docker_atom/work/script-setup-core.sh | 2 +- docker_atom/work/script-setup-sys.sh | 2 +- docker_docker_kit/work/script-setup-docker.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker_atom/work/script-setup-core.sh b/docker_atom/work/script-setup-core.sh index 3bb3e7fc..fe2e772d 100755 --- a/docker_atom/work/script-setup-core.sh +++ b/docker_atom/work/script-setup-core.sh @@ -214,7 +214,7 @@ setup_node_pnpm() { else URL_PNPM="https://github.com/pnpm/pnpm/releases/download/v${VER_PNPM}/pnpm-${UNAME}-${ARCH}" \ && echo "@ Installing pnpm v${VER_PNPM} from: ${URL_PNPM}" \ - && sudo curl_fsSL -o /opt/node/bin/pnpm "${URL_PNPM}" ; + && sudo curl -fsSL -o /opt/node/bin/pnpm "${URL_PNPM}" ; fi \ && sudo chmod +x /opt/node/bin/pnpm && sudo ln -sf /opt/node/bin/pnpm /usr/local/bin/pnpm \ && echo 'export PNPM_STORE_DIR=/opt/node/pnpm-store' | sudo tee /etc/profile.d/path-pnpm.sh >/dev/null ; diff --git a/docker_atom/work/script-setup-sys.sh b/docker_atom/work/script-setup-sys.sh index efccd51c..c86e0307 100755 --- a/docker_atom/work/script-setup-sys.sh +++ b/docker_atom/work/script-setup-sys.sh @@ -15,7 +15,7 @@ setup_tini() { && [ -n "${VER_TINI}" ] \ && local URL_TINI="https://github.com/krallin/tini/releases/download/v${VER_TINI}/tini-${ARCH}" \ && echo "Downloading Tini ${VER_TINI} from ${URL_TINI}" \ - && sudo curl_fsSL -o /usr/bin/tini "$URL_TINI" && sudo chmod +x /usr/bin/tini ; + && sudo curl -fsSL -o /usr/bin/tini "$URL_TINI" && sudo chmod +x /usr/bin/tini ; type tini && echo "@ Version of tini: $(tini --version)" || return 1 ; # ref: https://cloud-atlas.readthedocs.io/zh-cn/latest/docker/init/docker_tini.html # to run multi-proces with tini: use a bash script ends with the following code diff --git a/docker_docker_kit/work/script-setup-docker.sh b/docker_docker_kit/work/script-setup-docker.sh index 7adbf80d..fc38de06 100755 --- a/docker_docker_kit/work/script-setup-docker.sh +++ b/docker_docker_kit/work/script-setup-docker.sh @@ -6,11 +6,11 @@ setup_docker_compose() { && local VER_COMPOSE=$(curl_fsSL "https://api.github.com/repos/docker/compose/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[.\d]+') \ && local URL_COMPOSE="https://github.com/docker/compose/releases/download/v${VER_COMPOSE}/docker-compose-linux-${ARCH}" \ && echo "Downloading Compose from: ${URL_COMPOSE}" \ - && sudo curl_fsSL -o /usr/bin/docker-compose "${URL_COMPOSE}" \ + && sudo curl -fsSL -o /usr/bin/docker-compose "${URL_COMPOSE}" \ && sudo chmod +x /usr/bin/docker-compose ; type docker-compose && echo "@ Version of docker-compose: $(docker-compose --version)" || return 1 ; } - + setup_docker_syncer() { local ARCH=$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/') \ && local VER_SYNCER="$(curl_fsSL "https://api.github.com/repos/AliyunContainerService/image-syncer/releases?per_page=50" | grep -Po '(?<="tag_name": ")[^"]+' | head -1 | grep -Po '\d[.\d]+')" \ From c4c793203ed3fc2d7a01caf0a21d5d7ec10b9c6c Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Sun, 15 Nov 2026 08:37:15 +0000 Subject: [PATCH 8/8] debug img sync --- .github/workflows/build-docker-db.yml | 4 +- .github/workflows/build-docker-gpu.yml | 4 +- .github/workflows/build-docker.yml | 5 +- docker_docker_kit/README.md | 8 ++ docker_docker_kit/work/script-sync-image.sh | 105 ++++++++++++++++++-- tool.sh | 7 +- 6 files changed, 110 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-docker-db.yml b/.github/workflows/build-docker-db.yml index b76e285a..17f65012 100644 --- a/.github/workflows/build-docker-db.yml +++ b/.github/workflows/build-docker-db.yml @@ -23,9 +23,6 @@ env: # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. DOCKER_REGISTRY_USERNAME: ${{ vars.DOCKER_REGISTRY_USERNAME }} DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} - # used to sync image to mirror registry - DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }} - DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-foundation' }} jobs: @@ -56,6 +53,7 @@ jobs: - env: AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }} + DOCKER_MIRROR_TARGETS: ${{ vars.DOCKER_MIRROR_TARGETS }} run: | source ./tool.sh printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows diff --git a/.github/workflows/build-docker-gpu.yml b/.github/workflows/build-docker-gpu.yml index d39c7961..16714456 100644 --- a/.github/workflows/build-docker-gpu.yml +++ b/.github/workflows/build-docker-gpu.yml @@ -23,9 +23,6 @@ env: # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. DOCKER_REGISTRY_USERNAME: ${{ vars.DOCKER_REGISTRY_USERNAME }} DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} - # used to sync image to mirror registry - DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }} - DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-foundation' }} jobs: @@ -161,6 +158,7 @@ jobs: - env: AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }} + DOCKER_MIRROR_TARGETS: ${{ vars.DOCKER_MIRROR_TARGETS }} run: | source ./tool.sh printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index e572afe1..d25851f0 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -23,9 +23,6 @@ env: # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. DOCKER_REGISTRY_USERNAME: ${{ vars.DOCKER_REGISTRY_USERNAME }} DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} - # used to sync image to mirror registry - DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }} - DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-foundation' }} jobs: @@ -228,6 +225,7 @@ jobs: - env: AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }} + DOCKER_MIRROR_TARGETS: ${{ vars.DOCKER_MIRROR_TARGETS }} run: | source ./tool.sh && build_image docker-kit latest docker_docker_kit/docker-kit.Dockerfile && push_image docker-kit echo "jobs: { job-docker_kit: { name: 'docker-kit' } }" | tee ".github/workflows/build-docker.yml" > /dev/null @@ -245,6 +243,7 @@ jobs: - env: AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }} + DOCKER_MIRROR_TARGETS: ${{ vars.DOCKER_MIRROR_TARGETS }} run: | source ./tool.sh printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows diff --git a/docker_docker_kit/README.md b/docker_docker_kit/README.md index 531c354c..33ed0981 100644 --- a/docker_docker_kit/README.md +++ b/docker_docker_kit/README.md @@ -26,6 +26,14 @@ docker run --rm \ Fill the environment variables above, and more details can be found in `./work/script-sync-image.sh`. For workflow-based syncs, `IMG_NAMESPACE_SRC` selects the source namespace and `IMG_NAMESPACE_DST` optionally selects a different destination namespace; when omitted, the destination namespace matches the source. +For different destination namespaces per registry, set `DOCKER_MIRROR_TARGETS` to comma-separated `registry=namespace-template` entries: + +```shell +DOCKER_MIRROR_TARGETS='docker.io=%s,registry.cn-hangzhou.aliyuncs.com=%s-oss' +``` + +The single `%s` placeholder is replaced with `IMG_NAMESPACE`. In GitHub Actions, the main branch uses the repository owner namespace (for example, `labnow`), while all non-main branches use the fixed `0dev` suffix (for example, `labnow0dev`). The workflow syncs `quay.io//` to each configured target namespace. `DOCKER_MIRROR_TARGETS` takes precedence over the legacy `DOCKER_MIRROR_REGISTRY`; without it, legacy behavior is unchanged. Each destination registry must have matching credentials in the auth file. + ### Sync images in batch To sync images in batch, two config files (or combine them in one as `--config`) are needed. diff --git a/docker_docker_kit/work/script-sync-image.sh b/docker_docker_kit/work/script-sync-image.sh index 5c73d9ce..f2e14071 100644 --- a/docker_docker_kit/work/script-sync-image.sh +++ b/docker_docker_kit/work/script-sync-image.sh @@ -9,7 +9,9 @@ Usage: script-sync-image.sh [IMAGE] [options] --image-namespace-dst NAME Destination image namespace (default: source namespace) --source-registry NAME Source registry (default: REGISTRY_URL or docker.io) --target-registry NAME Target registry; repeat or use comma-separated values - --image IMAGE Sync one fully-qualified image path instead of workflow jobs + --image IMAGE Sync one registry-relative source image path + DOCKER_MIRROR_TARGETS Comma-separated registry=namespace-template entries. + A single %s is replaced with IMG_NAMESPACE. --help Show this help EOF } @@ -20,12 +22,15 @@ namespace_img_src=${IMG_NAMESPACE_SRC:-library} namespace_img_dst=${IMG_NAMESPACE_DST:-} registry_source=${REGISTRY_URL:-docker.io} registries_target=${DOCKER_MIRROR_REGISTRY:-} +mirror_targets=${DOCKER_MIRROR_TARGETS:-} +target_registry_cli=false image_single= option_value= option_shift= append_target_registries() { registries_target="${registries_target}${registries_target:+,}$1" + target_registry_cli=true } read_option_value() { @@ -84,9 +89,20 @@ while [ "$#" -gt 0 ]; do esac done +if [ -n "$mirror_targets" ] && [ "$target_registry_cli" = true ]; then + echo "Do not combine DOCKER_MIRROR_TARGETS with --target-registry" >&2 + exit 2 +fi + if [ -z "$namespace_img_dst" ]; then namespace_img_dst=$namespace_img_src fi +namespace_template_value=${IMG_NAMESPACE:-$namespace_img_dst} +use_target_map=false +[ -z "$mirror_targets" ] || use_target_map=true +if [ "$use_target_map" = true ] && [ -n "$registries_target" ]; then + echo "DOCKER_MIRROR_TARGETS overrides DOCKER_MIRROR_REGISTRY" >&2 +fi export SOURCE_REGISTRY=$registry_source @@ -99,6 +115,8 @@ trap 'exit 1' HUP INT TERM file_names=$tmp_dir/names file_names_raw=$tmp_dir/raw-names file_targets=$tmp_dir/targets +file_targets_raw=$tmp_dir/raw-targets +file_target_registries=$tmp_dir/target-registries file_config=$tmp_dir/config.json if [ -n "$image_single" ]; then @@ -115,10 +133,62 @@ else tr ',' '\n' < "$file_names_raw" > "$file_names" fi -printf '%s\n' "$registries_target" | tr ',' '\n' | - sed 's/^[[:space:]]*//; s/[[:space:]]*$//; /^$/d' > "$file_targets" -if [ ! -s "$file_targets" ]; then - printf '%s\n' registry.cn-beijing.aliyuncs.com registry.cn-hangzhou.aliyuncs.com > "$file_targets" +if [ "$use_target_map" = true ]; then + printf '%s\n' "$mirror_targets" | tr ',' '\n' | + sed 's/^[[:space:]]*//; s/[[:space:]]*$//; /^$/d' > "$file_targets_raw" + if [ ! -s "$file_targets_raw" ]; then + echo "DOCKER_MIRROR_TARGETS contains no target mappings" >&2 + exit 2 + fi + while IFS= read -r target_mapping || [ -n "$target_mapping" ]; do + case "$target_mapping" in + *=*) registry_target=${target_mapping%%=*}; namespace_template=${target_mapping#*=} ;; + *) echo "Invalid target mapping (expected registry=namespace): $target_mapping" >&2; exit 2 ;; + esac + registry_target=$(printf '%s' "$registry_target" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + namespace_template=$(printf '%s' "$namespace_template" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$registry_target" in + ''|*/*|*://*|*[[:space:]]*) echo "Invalid target registry: $registry_target" >&2; exit 2 ;; + esac + [ -n "$namespace_template" ] || { echo "Target namespace is empty for $registry_target" >&2; exit 2; } + + case "$namespace_template" in + *%s*) + namespace_template_prefix=${namespace_template%%\%s*} + namespace_template_suffix=${namespace_template#*%s} + case "$namespace_template_prefix$namespace_template_suffix" in + *%*|*[!a-zA-Z0-9._/-]*) + echo "Invalid namespace template (use only one %s placeholder): $namespace_template" >&2 + exit 2 + ;; + esac + [ -n "$namespace_template_value" ] || { + echo "IMG_NAMESPACE is empty; cannot expand target namespace: $namespace_template" >&2 + exit 2 + } + target_namespace=$(printf "$namespace_template" "$namespace_template_value") + ;; + *%*) echo "Unsupported printf placeholder in: $namespace_template" >&2; exit 2 ;; + *) target_namespace=$namespace_template ;; + esac + target_namespace=$(printf '%s' "$target_namespace" | tr '[:upper:]' '[:lower:]') + case "$target_namespace" in + ''|/*|*/|*//*|*[!a-z0-9._/-]*) + echo "Invalid target namespace: $target_namespace" >&2 + exit 2 + ;; + esac + printf '%s\t%s\n' "$registry_target" "$target_namespace" >> "$file_targets" + done < "$file_targets_raw" +else + printf '%s\n' "$registries_target" | tr ',' '\n' | + sed 's/^[[:space:]]*//; s/[[:space:]]*$//; /^$/d' > "$file_target_registries" + if [ ! -s "$file_target_registries" ]; then + printf '%s\n' registry.cn-beijing.aliyuncs.com registry.cn-hangzhou.aliyuncs.com > "$file_target_registries" + fi + while IFS= read -r registry_target || [ -n "$registry_target" ]; do + printf '%s\t%s\n' "$registry_target" "$namespace_img_dst" >> "$file_targets" + done < "$file_target_registries" fi status_total=0 @@ -127,20 +197,37 @@ run_image_sync() { status_total=$((status_total + $?)) } +image_destination_for_target() { + if [ "$use_target_map" = true ]; then + printf '%s/%s/%s' "$1" "$2" "$image_path_suffix" + else + printf '%s/%s' "$1" "$path_image_dst" + fi +} + while IFS= read -r image_name || [ -n "$image_name" ]; do path_image_src=$image_name path_image_dst=$image_name + image_path_suffix=$image_name if [ -z "$image_single" ]; then path_image_src=$namespace_img_src/$image_name path_image_dst=$namespace_img_dst/$image_name + elif [ "$use_target_map" = true ]; then + case "$image_single" in + "$namespace_img_src"/*) image_path_suffix=${image_single#"$namespace_img_src"/} ;; + *) + echo "With DOCKER_MIRROR_TARGETS, --image must start with source namespace '$namespace_img_src/'" >&2 + exit 2 + ;; + esac fi image_source=$registry_source/$path_image_src echo "Docker image sync job name found: $path_image_src" if [ -n "$file_auth" ]; then export SOURCE_IMAGE=$image_source TARGET_REGISTRIES= - while IFS= read -r registry_target || [ -n "$registry_target" ]; do - image_destination=$registry_target/$path_image_dst + while IFS="$(printf '\t')" read -r registry_target target_namespace || [ -n "$registry_target" ]; do + image_destination=$(image_destination_for_target "$registry_target" "$target_namespace") if [ -n "$TARGET_REGISTRIES" ]; then TARGET_REGISTRIES=$TARGET_REGISTRIES,$image_destination else @@ -163,8 +250,8 @@ while IFS= read -r image_name || [ -n "$image_name" ]; do echo "DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD are required without --file-auth" >&2 exit 254 fi - while IFS= read -r registry_target || [ -n "$registry_target" ]; do - image_destination=$registry_target/$path_image_dst + while IFS="$(printf '\t')" read -r registry_target target_namespace || [ -n "$registry_target" ]; do + image_destination=$(image_destination_for_target "$registry_target" "$target_namespace") export SOURCE_IMAGE=$image_source TARGET_REGISTRY=$registry_target DESTINATION=$image_destination if ! yq -n -o=json \ '{"images": {(strenv(SOURCE_IMAGE)): [strenv(DESTINATION)]}, "auth": {(strenv(TARGET_REGISTRY)): {"username": strenv(DOCKER_MIRROR_REGISTRY_USERNAME), "password": strenv(DOCKER_MIRROR_REGISTRY_PASSWORD)}}} * {"auth": {(strenv(SOURCE_REGISTRY)): {"username": strenv(DOCKER_REGISTRY_USERNAME), "password": strenv(DOCKER_REGISTRY_PASSWORD)}}}' \ diff --git a/tool.sh b/tool.sh index 8364b9da..fd8c41e2 100755 --- a/tool.sh +++ b/tool.sh @@ -13,11 +13,8 @@ else fi -CI_PROJECT_SPACE=${CI_PROJECT_BRANCH%%/*} - -# If on the main branch, image namespace will be same as CI_PROJECT_NAME's name space; -# else (not main branch), image namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE}. -[ "${CI_PROJECT_BRANCH}" = "main" ] && NAMESPACE_SUFFIX="" || NAMESPACE_SUFFIX="0${CI_PROJECT_SPACE}" ; +# On main, use the repository owner's namespace; on other branches, append the fixed "0dev" suffix. +[ "${CI_PROJECT_BRANCH}" = "main" ] && NAMESPACE_SUFFIX="" || NAMESPACE_SUFFIX="0dev" ; export CI_PROJECT_NAMESPACE="$(dirname "${CI_PROJECT_NAME}")${NAMESPACE_SUFFIX}" ; export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}')