diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..62265154b --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + +self-hosted-runner: + labels: + - ubuntu-slim diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 7933f62af..c6b086535 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + name: Release with goreleaser on: @@ -6,26 +10,47 @@ on: tags: - v* -permissions: write-all # Necessary for the generate-build-provenance action with containers +permissions: + contents: write + packages: write + attestations: write + id-token: write jobs: + go-version: + runs-on: ubuntu-slim + outputs: + version: ${{ steps.go-mod.outputs.version }} + steps: + - uses: actions/checkout@v6.0.3 + with: + persist-credentials: false + - id: go-mod + run: echo "version=$(awk '/^go / {print $2; exit}' go.mod)" >> "$GITHUB_OUTPUT" + stress-tests: - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v6.0.3 + with: + persist-credentials: false + - uses: actions/setup-go@v6.4.0 with: - go-version: stable + go-version-file: go.mod - name: Run release stress tests - run: go test -tags=stress -count=1 ./cmd/cloud-init-server ./pkg/wgtunnel ./internal/memstore ./internal/smdclient + run: make test-stress release: - needs: stress-tests + needs: + - go-version + - stress-tests uses: OpenCHAMI/github-actions/.github/workflows/go-build-release.yml@v3.2 with: cgo-enabled: "1" + go-version: ${{ needs.go-version.outputs.version }} + goreleaser-version: "v2.11.2" pre-build-commands: | - go install github.com/swaggo/swag/cmd/swag@latest + go install github.com/swaggo/swag/cmd/swag@v1.16.6 sudo apt update && sudo apt install -y git gcc g++ make ca-certificates curl gnupg gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu libc6-dev-arm64-cross software-properties-common clang-tools libstdc++-13-dev-arm64-cross attestation-binary-path: "dist/cloud-init*" registry-name: ghcr.io/openchami/cloud-init diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 22a11f118..4b87d9618 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,25 +1,57 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + name: Run golangci-lint + on: push: - branches: - - main + branches: [main] pull_request: + workflow_dispatch: + +concurrency: + group: lint-${{ github.ref }} + cancel-in-progress: true permissions: contents: read - # Optional: allow read access to pull requests. Use with `only-new-issues` option. - # pull-requests: read jobs: golangci: - name: lint - runs-on: ubuntu-latest + name: Run golangci-lint + runs-on: ubuntu-slim + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@v6.0.3 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v6.4.0 + with: + go-version-file: go.mod + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v9.2.0 + with: + version: v2.13.2 + + modules: + name: Check Go modules + runs-on: ubuntu-slim + timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - name: Checkout repository + uses: actions/checkout@v6.0.3 with: - go-version: stable - - name: golangci-lint - uses: golangci/golangci-lint-action@v8 + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v6.4.0 with: - version: v2.11 + go-version-file: go.mod + + - name: Check go.mod and go.sum + run: make tidy-check diff --git a/.github/workflows/security.yaml b/.github/workflows/security.yaml new file mode 100644 index 000000000..9964c8ebd --- /dev/null +++ b/.github/workflows/security.yaml @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + +name: Security + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: security-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + govulncheck: + name: govulncheck + uses: OpenCHAMI/github-actions/.github/workflows/govulncheck.yml@v3.2 + with: + go-package: ./... + + dependency-review: + name: dependency-review + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: write + uses: OpenCHAMI/github-actions/.github/workflows/dependency-review.yml@v3.2 + with: + fail-on-severity: high + comment-summary-in-pr: never + + workflow-lint: + name: workflow lint + permissions: + contents: read + security-events: write + uses: OpenCHAMI/github-actions/.github/workflows/lint-workflows.yml@v3.2 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..18f858aaf --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + +name: Test + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: test-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + unit: + name: Unit tests + runs-on: ubuntu-slim + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@v6.0.3 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v6.4.0 + with: + go-version-file: go.mod + + - name: Run tests + run: make test + + race: + name: Race tests + runs-on: ubuntu-slim + timeout-minutes: 20 + steps: + - name: Checkout repository + uses: actions/checkout@v6.0.3 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v6.4.0 + with: + go-version-file: go.mod + + - name: Run race tests + run: make test-race + + build: + name: Build binary + runs-on: ubuntu-slim + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@v6.0.3 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v6.4.0 + with: + go-version-file: go.mod + + - name: Build + run: make build diff --git a/.gitignore b/.gitignore index bfd0c7f58..6f6cb07e0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + dist +bin/ +graphify-out/ +pr_*_diff.txt +pr_*_view.txt # Our compiled binary /cloud-init-server .omo/ -graphify-out/cache/ \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..a226e517e --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + +version: "2" +run: + timeout: 5m + allow-parallel-runners: true +linters: + enable: + - govet + disable: + - staticcheck + - unused + - errcheck + settings: + exhaustive: + default-signifies-exhaustive: true + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - github.com/openchami/cloud-init + exclusions: + generated: lax diff --git a/.goreleaser-darwin.yaml b/.goreleaser-darwin.yaml index 0a0eb975d..1d8baa403 100644 --- a/.goreleaser-darwin.yaml +++ b/.goreleaser-darwin.yaml @@ -1,11 +1,13 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + version: 2 project_name: cloud-init before: hooks: - # You may remove this if you don't use go modules. - - go mod tidy - - go install github.com/swaggo/swag/cmd/swag@latest - - swag init -g cmd/cloud-init-server/main.go + - go install github.com/swaggo/swag/cmd/swag@v1.16.6 + - swag init -g cmd/cloud-init-server/main.go builds: - id: darwin @@ -39,4 +41,4 @@ builds: # The lines beneath this are called `modelines`. See `:help modeline` # Feel free to remove those if you don't want/use them. # yaml-language-server: $schema=https://raw.githubusercontent.com/goreleaser/goreleaser/v2.11.2/www/docs/static/schema.json -# vim: set ts=2 sw=2 tw=0 fo=cnqoj \ No newline at end of file +# vim: set ts=2 sw=2 tw=0 fo=cnqoj diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 4dc0f7ad8..202cbf5da 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,10 +1,12 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + version: 2 project_name: cloud-init before: hooks: - # You may remove this if you don't use go modules. - - go mod tidy - - go install github.com/swaggo/swag/cmd/swag@latest + - go install github.com/swaggo/swag/cmd/swag@v1.16.6 - swag init -g cmd/cloud-init-server/main.go builds: @@ -56,7 +58,7 @@ dockers: goamd64: v3 extra_files: - - LICENSE + - LICENSES/MIT.txt - CHANGELOG.md - README.md - image_templates: @@ -72,7 +74,7 @@ dockers: - "--label=org.opencontainers.image.revision={{.FullCommit}}" - "--label=org.opencontainers.image.version={{.Version}}" extra_files: - - LICENSE + - LICENSES/MIT.txt - CHANGELOG.md - README.md goarch: arm64 @@ -99,7 +101,8 @@ docker_manifests: - *arm64v8_linux_image archives: - - format: tar.gz + - formats: + - tar.gz # this name template makes the OS and Arch compatible with the results of uname. name_template: >- {{ .ProjectName }}_ @@ -110,7 +113,7 @@ archives: {{- if .Arm }}v{{ .Arm }}{{ end }} {{- if .Amd64 }}{{ .Amd64 }}{{ end }} files: - - LICENSE + - LICENSES/MIT.txt - CHANGELOG.md - README.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..8da7a3367 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# SPDX-License-Identifier: MIT +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + +- repo: https://github.com/fsfe/reuse-tool + rev: v5.1.1 + hooks: + - id: reuse-lint-file + +- repo: https://github.com/tekwizely/pre-commit-golang + # See 'pre-commit help autoupdate' + rev: v1.0.0-rc.2 + hooks: + + - id: go-mod-tidy + - id: go-test-mod + - id: go-vet-mod + + # + # Formatters + # + - id: go-fmt + - id: go-fmt-repo + + # + # Style Checkers + - id: golangci-lint-mod + + + # diff --git a/CHANGELOG.md b/CHANGELOG.md index 838dbe741..92ad0e4ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ + + # Changelog All notable changes to this project will be documented in this file. diff --git a/Dockerfile b/Dockerfile index 22bbe74cf..db3b790e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT # syntax=docker/dockerfile:1.7 FROM ubuntu:24.04 AS duckdb-ext ARG TARGETARCH @@ -26,7 +29,7 @@ RUN mkdir -p "$DUCKDB_HOME" # Preinstall whatever you need RUN duckdb -c "INSTALL 'json';" \ - && duckdb -c "INSTALL 'parquet';" + && duckdb -c "INSTALL 'parquet';" # -------- runtime image -------- FROM ubuntu:24.04 @@ -62,4 +65,3 @@ ENV LISTEN="0.0.0.0:27777" USER 65534:65534 ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/local/bin/cloud-init-server"] - diff --git a/Dockerfile.debug b/Dockerfile.debug index 12a0c1a69..b973803e0 100644 --- a/Dockerfile.debug +++ b/Dockerfile.debug @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT # Produces an image that runs cloud-init under a debugger. To use, build and push # to a registry with: # @@ -9,13 +12,11 @@ # # dlv connect container-hostname.example:2345 -FROM golang:1.24-bookworm +FROM golang:1.26.5-bookworm # Configure go env ENV GOPATH=/usr/local/golib -RUN export GOPATH=$GOPATH -RUN go env -w GO111MODULE=auto -RUN export CGO_ENABLED=0 +ENV CGO_ENABLED=1 # Copy source files COPY cmd $GOPATH/src/github.com/openchami/cloud-init/cmd @@ -33,7 +34,9 @@ RUN go build -C $GOPATH/src/github.com/openchami/cloud-init/cmd/cloud-init-serve RUN ln -s /usr/local/bin/cloud-init-server /cloud-init-server RUN go install github.com/go-delve/delve/cmd/dlv@v1.24.0 -RUN apt update; apt install tini +RUN apt-get update \ + && apt-get install -y --no-install-recommends tini \ + && rm -rf /var/lib/apt/lists/* USER 65534:65534 diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 6e7dcec43..000000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright © 2024 Triad National Security, LLC. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt new file mode 100644 index 000000000..881425ef6 --- /dev/null +++ b/LICENSES/MIT.txt @@ -0,0 +1,18 @@ +MIT License + +Copyright (c) 2026 OpenCHAMI a Series of LF Projects, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..70efedec7 --- /dev/null +++ b/Makefile @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + +.PHONY: help build clean docker-build fmt mod-install lint lint-fix release-check release-snapshot run test test-race test-stress tidy tidy-check vet vuln check + +GO ?= go +GIT ?= git +GOLANGCI_LINT ?= golangci-lint +GOVULNCHECK ?= govulncheck +CONTAINER_PROG ?= docker + +BINARY_NAME ?= cloud-init-server +CONTAINER_TAG ?= latest +TEST_TIMEOUT ?= 5m +STRESS_TEST_TIMEOUT ?= 10m +GOFLAGS ?= -v +GO_MOD_VERSION ?= $(shell awk '/^go / {print $$2; exit}' go.mod) +GO_TOOLCHAIN_VERSION ?= $(GO_MOD_VERSION) +GOTOOLCHAIN ?= go$(GO_TOOLCHAIN_VERSION) +GO_ENV := GOTOOLCHAIN=$(GOTOOLCHAIN) +GORELEASER_VERSION ?= v2.11.2 +GORELEASER ?= $(GO_ENV) $(GO) run github.com/goreleaser/goreleaser/v2@$(GORELEASER_VERSION) + +VERSION ?= $(shell $(GIT) describe --tags --always --dirty 2>/dev/null || echo dev) +COMMIT ?= $(shell $(GIT) rev-parse --short HEAD 2>/dev/null || echo unknown) +DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") +GIT_BRANCH ?= $(shell $(GIT) rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown) +GIT_TAG ?= $(shell $(GIT) describe --tags --abbrev=0 2>/dev/null || echo unknown) +GIT_STATE ?= $(shell if $(GIT) diff-index --quiet HEAD -- 2>/dev/null; then echo clean; else echo dirty; fi) +BUILD_HOST ?= $(shell hostname) +GO_VERSION ?= $(shell $(GO_ENV) $(GO) env GOVERSION 2>/dev/null || echo unknown) +BUILD_USER ?= $(shell whoami) + +LDFLAGS := -ldflags "-X 'main.GitCommit=$(COMMIT)' \ + -X 'main.BuildTime=$(DATE)' \ + -X 'main.Version=$(VERSION)' \ + -X 'main.GitBranch=$(GIT_BRANCH)' \ + -X 'main.GitTag=$(GIT_TAG)' \ + -X 'main.GitState=$(GIT_STATE)' \ + -X 'main.BuildHost=$(BUILD_HOST)' \ + -X 'main.GoVersion=$(GO_VERSION)' \ + -X 'main.BuildUser=$(BUILD_USER)'" + +help: ## Display this help screen + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m[VAR=val]... \033[0m\n\nTargets:\n"} /^[a-zA-Z0-9_\/.-]+:.*##/ {printf " \033[36m%-22s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +mod-install: ## Download and verify Go modules + $(GO_ENV) $(GO) mod download + $(GO_ENV) $(GO) mod verify + +build: ## Build the cloud-init server binary + mkdir -p bin + $(GO_ENV) $(GO) build $(GOFLAGS) $(LDFLAGS) -o bin/$(BINARY_NAME) ./cmd/cloud-init-server + +run: build ## Build and run the service + ./bin/$(BINARY_NAME) + +test: ## Run unit tests without the race detector + $(GO_ENV) $(GO) test $(GOFLAGS) -timeout $(TEST_TIMEOUT) -count=1 -shuffle=on ./... + +test-race: ## Run unit tests with the race detector + $(GO_ENV) GORACE="halt_on_error=1" $(GO) test $(GOFLAGS) -timeout $(TEST_TIMEOUT) -race -count=1 -shuffle=on ./... + +test-stress: ## Run release stress tests + $(GO_ENV) $(GO) test $(GOFLAGS) -tags=stress -timeout $(STRESS_TEST_TIMEOUT) -count=1 ./cmd/cloud-init-server ./pkg/wgtunnel ./internal/memstore ./internal/smdclient + +tidy: ## Tidy go.mod and go.sum + $(GO_ENV) $(GO) mod tidy + +tidy-check: ## Verify go.mod and go.sum are tidy + @tmpdir=$$(mktemp -d); \ + cp go.mod "$$tmpdir/go.mod"; \ + cp go.sum "$$tmpdir/go.sum"; \ + $(GO_ENV) $(GO) mod tidy; \ + if ! cmp -s go.mod "$$tmpdir/go.mod" || ! cmp -s go.sum "$$tmpdir/go.sum"; then \ + echo "go.mod or go.sum changed after go mod tidy"; \ + rm -rf "$$tmpdir"; \ + exit 1; \ + fi; \ + rm -rf "$$tmpdir" + +fmt: ## Format Go source + $(GO_ENV) $(GO) fmt ./... + +vet: ## Run go vet + $(GO_ENV) $(GO) vet ./... + +lint: ## Run golangci-lint + $(GO_ENV) $(GOLANGCI_LINT) run ./... + +lint-fix: ## Run golangci-lint with autofix + $(GO_ENV) $(GOLANGCI_LINT) run --fix ./... + +vuln: ## Run govulncheck + $(GO_ENV) $(GOVULNCHECK) ./... + +docker-build: ## Build the runtime container image locally + $(CONTAINER_PROG) build -t $(BINARY_NAME):$(CONTAINER_TAG) . + +release-check: ## Validate GoReleaser configuration + $(GORELEASER) check + +release-snapshot: ## Build a local GoReleaser snapshot without publishing + $(GO_ENV) GIT_STATE=$(GIT_STATE) BUILD_HOST=$(BUILD_HOST) GO_VERSION=$(GO_VERSION) BUILD_USER=$(BUILD_USER) \ + $(GORELEASER) release --snapshot --clean --skip=publish + +clean: ## Clean local build artifacts + rm -rf bin dist coverage.out coverage.html + $(GO_ENV) $(GO) clean -cache + +check: tidy-check lint test-race build vuln ## Run local pre-PR checks + +.DEFAULT_GOAL := help diff --git a/README.md b/README.md index 0c6170fb2..79d0e3ea3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ + + # OpenCHAMI Cloud-Init Server ## Summary of Repo @@ -7,6 +13,7 @@ The **OpenCHAMI cloud-init service** retrieves detailed inventory information fr 1. [About / Introduction](#about--introduction) 2. [Build / Install](#build--install) - [Environment Variables](#environment-variables) + - [Local Automation](#local-automation) - [Building Locally with GoReleaser](#building-locally-with-goreleaser) 3. [Running the Service](#running-the-service) - [Cluster Name](#cluster-name) @@ -49,13 +56,32 @@ Cloud-init on nodes retrieves data in a fixed order: This project uses **[GoReleaser](https://goreleaser.com/)** for building and releasing, embedding additional metadata such as commit info, build time, and version. Below is a brief overview for local builds. +### Local Automation + +The repository provides a `Makefile` that mirrors the GitHub Actions quality gates. Common targets are: + +```bash +make build # build bin/cloud-init-server +make test # run unit tests once with shuffled order +make test-race # run tests with the race detector +make test-stress # run release stress tests +make tidy-check # verify go.mod and go.sum are tidy +make lint # run golangci-lint +make vuln # run govulncheck +make check # run the local pre-PR gate +make release-check # validate GoReleaser configuration +make release-snapshot # build a local snapshot without publishing +``` + +The CI and release workflows read the Go toolchain from `go.mod` instead of using `stable`, so local builds and GitHub Actions use the same Go version. + ### Environment Variables To include detailed metadata in your builds, set the following: -- **GIT_STATE**: `clean` if your repo is clean, `dirty` if uncommitted changes exist -- **BUILD_HOST**: Hostname of the build machine -- **GO_VERSION**: Version of Go used (for consistent versioning info) -- **BUILD_USER**: Username of the person/system performing the build +- **GIT_STATE**: `clean` if your repo is clean, `dirty` if uncommitted changes exist +- **BUILD_HOST**: Hostname of the build machine +- **GO_VERSION**: Version of Go used (for consistent versioning info) +- **BUILD_USER**: Username of the person/system performing the build ```bash export GIT_STATE=$(if git diff-index --quiet HEAD --; then echo 'clean'; else echo 'dirty'; fi) @@ -65,7 +91,7 @@ export BUILD_USER=$(whoami) ``` ### Building Locally with GoReleaser -1. [Install GoReleaser](https://goreleaser.com/install/) following their documentation. +1. [Install GoReleaser](https://goreleaser.com/install/) following their documentation. 2. Run in snapshot mode to build locally without releasing: ```bash @@ -226,7 +252,7 @@ curl -X PUT http://localhost:27777/cloud-init/admin/instance-info/x3000c1b1n1 \ ## More Reading -- [Official cloud-init documentation](https://cloud-init.io/) -- [OpenCHAMI TPM-manager service](https://github.com/OpenCHAMI/TPM-manager) -- [GoReleaser Documentation](https://goreleaser.com/) -- [SMD Documentation](https://github.com/OpenCHAMI/smd) +- [Official cloud-init documentation](https://cloud-init.io/) +- [OpenCHAMI TPM-manager service](https://github.com/OpenCHAMI/TPM-manager) +- [GoReleaser Documentation](https://goreleaser.com/) +- [SMD Documentation](https://github.com/OpenCHAMI/smd) diff --git a/REUSE.toml b/REUSE.toml new file mode 100644 index 000000000..0551130c7 --- /dev/null +++ b/REUSE.toml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + +version = 1 +SPDX-PackageName = "cloud-init" +SPDX-PackageSupplier = "OpenCHAMI " +SPDX-PackageDownloadLocation = "https://github.com/OpenCHAMI/cloud-init" + +[[annotations]] +path = [ + "docs/swagger.json", + "go.sum", + "tests/bss/bss_add.json", + "tests/harbor/add_entry.json", + "tests/harbor/compute_group.json", + "tests/harbor/test_compute1.json", + "tests/harbor/test_compute2.json", +] +precedence = "override" +SPDX-FileCopyrightText = "Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC" +SPDX-License-Identifier = "MIT" diff --git a/build-in-container.sh b/build-in-container.sh index 73c3586fb..7d141c10e 100755 --- a/build-in-container.sh +++ b/build-in-container.sh @@ -1,4 +1,10 @@ -#!/bin/bash +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + +set -euo pipefail # This script uses the latest Ubuntu 24.04 container to build the project with GoReleaser. It emulates the GitHub Actions environment as closely as possible. # Before submitting a PR for release/build. please run this script to ensure your PR will pass the build. @@ -8,14 +14,16 @@ CONTAINER_NAME="goreleaser-build" # Directory where built binaries will be available OUTPUT_DIR="$(pwd)/dist" +GO_VERSION="${GO_VERSION:-$(awk '/^go / {print $2; exit}' go.mod)}" +GORELEASER_VERSION="${GORELEASER_VERSION:-v2.11.2}" export GIT_STATE=$(if git diff-index --quiet HEAD --; then echo 'clean'; else echo 'dirty'; fi) export BUILD_HOST=$(hostname) -export GO_VERSION=$(go version | awk '{print $3}') +export GO_VERSION export BUILD_USER=$(whoami) # Start a new disposable Ubuntu 24.04 container with the current directory mounted -${CONTAINER_CMD:-docker} run --rm -it \ +${CONTAINER_CMD:-docker} run --rm \ --name "$CONTAINER_NAME" \ -v "$(pwd)":/workspace \ -v ${CONTAINER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock \ @@ -47,8 +55,7 @@ ${CONTAINER_CMD:-docker} run --rm -it \ apt update && apt install -y \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - # Install Go (match GitHub runner version) - curl -fsSL https://golang.org/dl/go1.21.5.linux-amd64.tar.gz | tar -C /usr/local -xz + curl -fsSL https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xz export PATH=\$PATH:/usr/local/go/bin go version # Verify Go installation @@ -57,8 +64,7 @@ ${CONTAINER_CMD:-docker} run --rm -it \ export PATH=\$PATH:\$GOPATH/bin echo \"GOPATH: \$GOPATH\" && echo \"PATH: \$PATH\" - # Install Goreleaser - curl -sL https://github.com/goreleaser/goreleaser/releases/latest/download/goreleaser_Linux_x86_64.tar.gz | tar -xz -C /usr/local/bin + curl -sL https://github.com/goreleaser/goreleaser/releases/download/${GORELEASER_VERSION}/goreleaser_Linux_x86_64.tar.gz | tar -xz -C /usr/local/bin goreleaser --version # Verify Goreleaser installation # Setup Docker buildx for multi-platform builds @@ -69,14 +75,14 @@ ${CONTAINER_CMD:-docker} run --rm -it \ export GIT_STATE="$GIT_STATE" export BUILD_HOST="$BUILD_HOST" export BUILD_USER="$BUILD_USER" - export GO_VERSION=$(go version | awk '{print $3}') + export GO_VERSION=\$(go version | awk '{print \$3}') # Convince git that our directory is safe git config --global --add safe.directory /workspace # Run Goreleaser - goreleaser release --snapshot --clean --skip archive,publish + goreleaser release --snapshot --clean --skip=publish " # Notify user of success -echo "✅ Build complete! Check the output in: $OUTPUT_DIR" +echo "Build complete. Check the output in: $OUTPUT_DIR" diff --git a/cmd/cloud-init-server/auth.go b/cmd/cloud-init-server/auth.go index 5a52dfcbc..bb89f4209 100644 --- a/cmd/cloud-init-server/auth.go +++ b/cmd/cloud-init-server/auth.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main // Adapted from OpenCHAMI SMD's auth.go diff --git a/cmd/cloud-init-server/base64_decode_test.go b/cmd/cloud-init-server/base64_decode_test.go index 8151a2eba..b926b0d42 100644 --- a/cmd/cloud-init-server/base64_decode_test.go +++ b/cmd/cloud-init-server/base64_decode_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import ( diff --git a/cmd/cloud-init-server/group_handlers.go b/cmd/cloud-init-server/group_handlers.go index 6040571f7..9cb4985e7 100644 --- a/cmd/cloud-init-server/group_handlers.go +++ b/cmd/cloud-init-server/group_handlers.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import ( @@ -6,6 +10,7 @@ import ( "reflect" "github.com/go-chi/chi/v5" + "github.com/openchami/cloud-init/pkg/cistore" ) diff --git a/cmd/cloud-init-server/handlers.go b/cmd/cloud-init-server/handlers.go index bebf91264..2f1a405c2 100644 --- a/cmd/cloud-init-server/handlers.go +++ b/cmd/cloud-init-server/handlers.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import ( @@ -6,12 +10,13 @@ import ( "net/http" "github.com/go-chi/chi/v5" + "github.com/rs/zerolog/log" + "github.com/swaggo/swag" + // Import to run swag.Register() to generated docs _ "github.com/openchami/cloud-init/docs" "github.com/openchami/cloud-init/internal/smdclient" "github.com/openchami/cloud-init/pkg/cistore" - "github.com/rs/zerolog/log" - "github.com/swaggo/swag" ) type CiHandler struct { diff --git a/cmd/cloud-init-server/main.go b/cmd/cloud-init-server/main.go index 5f30e499e..83150b614 100644 --- a/cmd/cloud-init-server/main.go +++ b/cmd/cloud-init-server/main.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main // @Title OpenCHAMI Cloud-Init Server API @@ -19,19 +23,19 @@ import ( "github.com/go-chi/chi/v5/middleware" openchami_authenticator "github.com/openchami/chi-middleware/auth" openchami_logger "github.com/openchami/chi-middleware/log" - "github.com/openchami/cloud-init/internal/memstore" - openchami_middleware "github.com/openchami/cloud-init/internal/middleware" - "github.com/openchami/cloud-init/internal/quackstore" - "github.com/openchami/cloud-init/internal/smdclient" - "github.com/openchami/cloud-init/pkg/cistore" - "github.com/openchami/cloud-init/pkg/wgtunnel" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/rs/zerolog/pkgerrors" - "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" + + "github.com/openchami/cloud-init/internal/memstore" + openchami_middleware "github.com/openchami/cloud-init/internal/middleware" + "github.com/openchami/cloud-init/internal/quackstore" + "github.com/openchami/cloud-init/internal/smdclient" + "github.com/openchami/cloud-init/pkg/cistore" + "github.com/openchami/cloud-init/pkg/wgtunnel" ) var ( diff --git a/cmd/cloud-init-server/metadata.go b/cmd/cloud-init-server/metadata.go index 50fcf67a0..27c113b56 100644 --- a/cmd/cloud-init-server/metadata.go +++ b/cmd/cloud-init-server/metadata.go @@ -1,10 +1,15 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import ( "fmt" - "github.com/openchami/cloud-init/pkg/cistore" "github.com/rs/zerolog/log" + + "github.com/openchami/cloud-init/pkg/cistore" ) type MetaData struct { diff --git a/cmd/cloud-init-server/metadata_handlers.go b/cmd/cloud-init-server/metadata_handlers.go index b7c20d9ba..f594553e7 100644 --- a/cmd/cloud-init-server/metadata_handlers.go +++ b/cmd/cloud-init-server/metadata_handlers.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import ( @@ -6,10 +10,11 @@ import ( "strings" "github.com/go-chi/chi/v5" - "github.com/openchami/cloud-init/internal/smdclient" - "github.com/openchami/cloud-init/pkg/cistore" "github.com/rs/zerolog/log" yaml "gopkg.in/yaml.v2" + + "github.com/openchami/cloud-init/internal/smdclient" + "github.com/openchami/cloud-init/pkg/cistore" ) func getActualRequestIP(r *http.Request) string { diff --git a/cmd/cloud-init-server/metadata_test.go b/cmd/cloud-init-server/metadata_test.go index 9b0cc8660..d74ab3e18 100644 --- a/cmd/cloud-init-server/metadata_test.go +++ b/cmd/cloud-init-server/metadata_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import ( @@ -5,6 +9,7 @@ import ( "testing" base "github.com/Cray-HPE/hms-base" + "github.com/openchami/cloud-init/pkg/cistore" ) diff --git a/cmd/cloud-init-server/peer_removal_queue.go b/cmd/cloud-init-server/peer_removal_queue.go index 077b9f18c..71cf1fcf7 100644 --- a/cmd/cloud-init-server/peer_removal_queue.go +++ b/cmd/cloud-init-server/peer_removal_queue.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import "github.com/rs/zerolog/log" diff --git a/cmd/cloud-init-server/peer_removal_queue_stress_test.go b/cmd/cloud-init-server/peer_removal_queue_stress_test.go index f1c0e4c35..45e72ad8a 100644 --- a/cmd/cloud-init-server/peer_removal_queue_stress_test.go +++ b/cmd/cloud-init-server/peer_removal_queue_stress_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + //go:build stress package main diff --git a/cmd/cloud-init-server/peer_removal_queue_test.go b/cmd/cloud-init-server/peer_removal_queue_test.go index 914901916..e5ab9559b 100644 --- a/cmd/cloud-init-server/peer_removal_queue_test.go +++ b/cmd/cloud-init-server/peer_removal_queue_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import ( @@ -8,6 +12,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/openchami/cloud-init/internal/smdclient" ) diff --git a/cmd/cloud-init-server/userdata_handlers.go b/cmd/cloud-init-server/userdata_handlers.go index 6fad9bdb5..f81aba7a4 100644 --- a/cmd/cloud-init-server/userdata_handlers.go +++ b/cmd/cloud-init-server/userdata_handlers.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import ( @@ -6,9 +10,10 @@ import ( "net/http" "github.com/go-chi/chi/v5" + "github.com/rs/zerolog/log" + "github.com/openchami/cloud-init/internal/smdclient" "github.com/openchami/cloud-init/pkg/cistore" - "github.com/rs/zerolog/log" ) // UserDataHandler godoc diff --git a/cmd/cloud-init-server/vendordata_handlers.go b/cmd/cloud-init-server/vendordata_handlers.go index e247b63f6..1dc1d9b67 100644 --- a/cmd/cloud-init-server/vendordata_handlers.go +++ b/cmd/cloud-init-server/vendordata_handlers.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import ( @@ -5,9 +9,10 @@ import ( "net/http" "github.com/go-chi/chi/v5" + "github.com/rs/zerolog/log" + "github.com/openchami/cloud-init/internal/smdclient" "github.com/openchami/cloud-init/pkg/cistore" - "github.com/rs/zerolog/log" ) // VendorDataHandler godoc diff --git a/cmd/cloud-init-server/version.go b/cmd/cloud-init-server/version.go index 66b7f42ac..2e3601ac1 100644 --- a/cmd/cloud-init-server/version.go +++ b/cmd/cloud-init-server/version.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +// +// SPDX-License-Identifier: MIT + package main import ( diff --git a/demo/Demo.md b/demo/Demo.md index d8084ecbe..1cb1e32fa 100644 --- a/demo/Demo.md +++ b/demo/Demo.md @@ -1,3 +1,9 @@ + + # Demo of new cloud-init behavior Updates to the cloud-init service are designed to push the complexity of merging configurations into the cloud-init client rather than the cloud-init server. The codepaths for reducing surprises in merge behavior are much better tested in the client which is open source and deployed on countless instances around the world. Staying compliant with the [nocloud-net datasource](https://cloudinit.readthedocs.io/en/latest/reference/datasources/nocloud.html) client requires a short review of how it handles the order of requests and the order of processing. @@ -184,5 +190,3 @@ curl -X PUT http://localhost:27777/cloud-init/admin/instance-info/x3000c1b1n1 \ "instance-type": "t2.micro" }' ``` - - diff --git a/demo/add_cabinet_groups.sh b/demo/add_cabinet_groups.sh index 2c3c069e1..adb4dfaf5 100755 --- a/demo/add_cabinet_groups.sh +++ b/demo/add_cabinet_groups.sh @@ -1,5 +1,9 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + curl -X POST http://localhost:27777/cloud-init/admin/groups/ \ -H "Content-Type: application/json" \ -d '{ @@ -51,4 +55,3 @@ curl -X POST http://localhost:27777/cloud-init/admin/groups/ \ "encoding": "plain" } }' - \ No newline at end of file diff --git a/demo/add_compute_group.sh b/demo/add_compute_group.sh index a717faf6c..be0a5c74b 100755 --- a/demo/add_compute_group.sh +++ b/demo/add_compute_group.sh @@ -1,5 +1,9 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + # Define the cloud-config content CLOUD_CONFIG_CONTENT=$(cat < + +# cloud-init +Cloud-init supports a myriad of [Data Sources](https://cloudinit.readthedocs.io/en/latest/reference/datasources.html) but so far we have been using [nocloud](https://cloudinit.readthedocs.io/en/latest/reference/datasources/nocloud.html). This allows us to use kernel cmdline parameters to set the endpoints for `cloud-init` to pull it's configs from. It's [documented](https://cloudinit.readthedocs.io/en/latest/reference/datasources/nocloud.html#source-files) that the cloud-init client will query a set of urls in a predictable order and then merge the data according to a predictable set of rules: 1. http://endpoint/meta-data @@ -8,10 +14,10 @@ It's [documented](https://cloudinit.readthedocs.io/en/latest/reference/datasourc 1. http://endpoint/network-config (only if indicated in user-data or vendor-data) For merging user-data and vendor-data, cloud-init follows the principle that user-supplied configurations (like user-data) should override vendor defaults (like vendor-data). For instance, if a root user is defined in both the vendor-data and the user-data, the one in the user-data will be created and the one in vendor-data will be ignored. - + ## background -The idea behind the cloud-init microservice is to provide `user-data`, `vendor-data`, and `meta-data` to a node during post-boot. The contents of the `*-data` payloads should be generated by the cloud-init server based on information provided by the sysadmins as well as information from SMD. +The idea behind the cloud-init microservice is to provide `user-data`, `vendor-data`, and `meta-data` to a node during post-boot. The contents of the `*-data` payloads should be generated by the cloud-init server based on information provided by the sysadmins as well as information from SMD. @@ -22,9 +28,9 @@ name: IDENTIFIER cloud-init: userdata: metadata: - vendordata: + vendordata: ``` -Where the contents of `userdata`, `metadata`, and `vendordata` are dictionaries and no restrictions are made, including whether it is valid cloud-init data or not. You can only add a single `cloud-init` data per `IDENTIFIER`, but there are no restrictions on the number of `IDENTIFIER`s. +Where the contents of `userdata`, `metadata`, and `vendordata` are dictionaries and no restrictions are made, including whether it is valid cloud-init data or not. You can only add a single `cloud-init` data per `IDENTIFIER`, but there are no restrictions on the number of `IDENTIFIER`s. These are stored in memory using the `MemStore` struct ```golang @@ -33,7 +39,7 @@ type MemStore struct { } ``` And `citypes` is like so: -```golang +```golang package citypes type CI struct { @@ -47,7 +53,7 @@ type CIData struct { VendorData map[string]interface{} `json:"vendordata"` } ``` -When a cloud-init client makes a request to the cloud-init server, the server will attempt to identify the requesting client by looking up it's IP address in the `/EthernetInterfaces` SMD endpoint and use the mapped `ComponentID`. The server will then try to find all the SMD groups this component is a member using `/memberships/{ComponentID}`. +When a cloud-init client makes a request to the cloud-init server, the server will attempt to identify the requesting client by looking up it's IP address in the `/EthernetInterfaces` SMD endpoint and use the mapped `ComponentID`. The server will then try to find all the SMD groups this component is a member using `/memberships/{ComponentID}`. Once a list of groups is found it will iterate over the groups and check to see if there is an IDENTIFIER that matches the group name. @@ -60,13 +66,13 @@ mergedMaps := lo.Assign( // map[string]int{"a": 1, "b": 3, "c": 4} ``` -This is not ideal because if a `ComponentID` is a member of multiple groups, and those groups share top-level keys (i.e. `write_files`), the returned `*-data` will only include the last group's data, overwriting previous group values. +This is not ideal because if a `ComponentID` is a member of multiple groups, and those groups share top-level keys (i.e. `write_files`), the returned `*-data` will only include the last group's data, overwriting previous group values. ### Current Main In the latest [main](https://github.com/openchami/cloud-init) branch of cloud-init, we have made significant changes to how the `*-data` structures are generated. -First, the payloads for populating the cloud-init server have changed endpoints and structure. Endpoints are now `/cloud-init/groups/{IDENTIFIER}` and the structure changes on whether we are adding `user-data` or `meta-data`. +First, the payloads for populating the cloud-init server have changed endpoints and structure. Endpoints are now `/cloud-init/groups/{IDENTIFIER}` and the structure changes on whether we are adding `user-data` or `meta-data`. `meta-data` looks like so: ```json { @@ -88,9 +94,9 @@ First, the payloads for populating the cloud-init server have changed endpoints } } ``` -The *only* supported key under `user-data` here is `write_files`, all other keys will be ignored. This not ideal for a lot of reasons and I think we are all in agreement on this. +The *only* supported key under `user-data` here is `write_files`, all other keys will be ignored. This not ideal for a lot of reasons and I think we are all in agreement on this. -The lookup behavior is, I believe, the same as before. The server will lookup the clients IP, get a `ComponentID`, then find all the SMD groups this `ComponentID` is a member of. After this the behavior changes. +The lookup behavior is, I believe, the same as before. The server will lookup the clients IP, get a `ComponentID`, then find all the SMD groups this `ComponentID` is a member of. After this the behavior changes. For `meta-data`, and new `meta-data` stucture is built where all the group specific `meta-data` content is bundled into the new stucture with the following format: ```yaml @@ -103,7 +109,7 @@ groups: IDENTIFIER2: key: value ``` -Where `IDENTIFIER1` and `IDENTIFIER2` are hypotheitcal groups a component is a member of AND those groups have stored `meta-data`. For the most part, I think this is a perfectly fine way to merge `meta-data` from multiple groups. +Where `IDENTIFIER1` and `IDENTIFIER2` are hypotheitcal groups a component is a member of AND those groups have stored `meta-data`. For the most part, I think this is a perfectly fine way to merge `meta-data` from multiple groups. For `user-data`, a new `user-data` structure is also built but the `user-data` format the cloud-init client expects is more restrictive than `meta-data`. It MUST look like something like ```yaml @@ -111,7 +117,7 @@ For `user-data`, a new `user-data` structure is also built but the `user-data` f write_files: [] runcmd: [] ``` -In this case the server makes a new `write_files` list that appends group specific `write_files`. +In this case the server makes a new `write_files` list that appends group specific `write_files`. For example the payload for `IDENTIFIER1` ```json { @@ -138,7 +144,7 @@ and the payload for `IDENTIDIER2` } } ``` -made to `/cloud-init/groups/IDENTIFIER1` and `/cloud-init/groups/IDENTIFIER2` respectively. +made to `/cloud-init/groups/IDENTIFIER1` and `/cloud-init/groups/IDENTIFIER2` respectively. When a component that is a member of `IDENTIFIER1` and `IDENTIFIER2` makes a request to the cloud-init server it will return the following: ```yaml @@ -151,15 +157,15 @@ write_files: ``` This is *good* behavior aside from the fact that ONLY `write_files` is supported. -I do not think that `vendor-data` is supported at all in this case. +I do not think that `vendor-data` is supported at all in this case. ### Travis/David Proposal What I am proposing is that we combine the two ways of building the cloud-init `*-data` payloads. -The v0.1.1 payload structure is the most clear to me. It includes the `IDENTIFIER` in the payload, so if you save these as files it is clear what you are adding data to versus the `/cloud-init/groups/IDENTIFIER` endpoint. It allows to you to add `user-data`, `meta-data`, and `vendor-data` (and eventually `network-config` if we ever figure that out) in a single payload. This has the benefit of being able to support a trivial cloud-init setup where there is no desire to have multiple groups per `ComponentID`. You can have a single cloud-init payload and things will work as expected. +The v0.1.1 payload structure is the most clear to me. It includes the `IDENTIFIER` in the payload, so if you save these as files it is clear what you are adding data to versus the `/cloud-init/groups/IDENTIFIER` endpoint. It allows to you to add `user-data`, `meta-data`, and `vendor-data` (and eventually `network-config` if we ever figure that out) in a single payload. This has the benefit of being able to support a trivial cloud-init setup where there is no desire to have multiple groups per `ComponentID`. You can have a single cloud-init payload and things will work as expected. -v0.1.1 is bad at the merge steps. Overwriting data depending on the ordering of the groups is not good behavior and it is also not clear that is what is happening. The current main branch of cloud-init handles this more clearly. -The `meta-data` has the same issue where conflicting keys will be overwritten by a latter group. +v0.1.1 is bad at the merge steps. Overwriting data depending on the ordering of the groups is not good behavior and it is also not clear that is what is happening. The current main branch of cloud-init handles this more clearly. +The `meta-data` has the same issue where conflicting keys will be overwritten by a latter group. The behavior of the main branch cloud-init is more robust in the meta-data area. Generating a meta-data that includes injected data from SMD and other generated data is good, and populating a `groups: {}` dictionary with all the meta-data from all groups allows for a lot of flexibility. The `user-data` for `write_files` is something I would like to see happen for all cloud-init `modules`. This leaves the complexity up to the user and makes it so we can support future cloud-init module additions without us having to really do anything. @@ -172,14 +178,14 @@ For example, assume I have a component that is a member of the following groups: - chrony - domain1 -SSH, rsyslog, and chrony are intended to be functional things, i.e. actions cloud-init should do on boot. -I should be able to upload cloud-configs for each of these functions. +SSH, rsyslog, and chrony are intended to be functional things, i.e. actions cloud-init should do on boot. +I should be able to upload cloud-configs for each of these functions. SSH: ```yaml name: ssh cloud-init: userdata: - write_files: + write_files: - content: {{ ds.meta_data.groups.domain1.ssh_root_pub_key }} path: /root/.ssh/authorized_keys ``` @@ -189,7 +195,7 @@ name: rsyslog cloud-init: userdata: write_files: - - content: | + - content: | module(load="impstats" interval="60" severity="7") module(load="imuxsock" SysSock.Use="on") include(file="/etc/rsyslog.d/*.conf" mode="optional") @@ -212,7 +218,7 @@ Chrony: name: chrony cloud-init: userdata: - write_files: + write_files: - content: | server {{ ds.meta_data.groups.domain1.chrony_server }} iburst driftfile /var/lib/chrony/drift @@ -226,7 +232,7 @@ cloud-init: - systemctl restart chronyd ``` -The `domain1` group is intended to be a group that stores variables. +The `domain1` group is intended to be a group that stores variables. ```yaml name: domain1 cloud-init: @@ -257,7 +263,7 @@ write_files: local7.* /var/log/boot.log action(type="omfwd" target="{{ ds.meta_data.groups.domain1.syslog_aggregator }}" port="514" protocol="tcp") path: /etc/rsyslog.conf - - content: | + - content: | server {{ ds.meta_data.groups.domain1.chrony_server }} iburst driftfile /var/lib/chrony/drift makestep 1.0 3 @@ -298,11 +304,11 @@ The `meta-data` will show in the instance-data.json under ``` Which is how we can use it in the `user-data` templates. -All of this is completely opt-in, so we don't require this to be broken up into pieces. It should be completely possible to combine all of this into a single payload. +All of this is completely opt-in, so we don't require this to be broken up into pieces. It should be completely possible to combine all of this into a single payload. But, by being able to break things up I think we can start thinking of really complex workflows that will allow us to build up nodes in building blocks. -Once scenario I don't have configs for is swapping a node from `slurm` to a `kubernetes` worker. I think we could do this with different groups containing `user-data` with functionality to configure `slurm` versus `kubernetes`, but also retain generic things like `ssh`, `chrony`, `rsyslog`, and lots of other basic things you'd want in any configuration. +Once scenario I don't have configs for is swapping a node from `slurm` to a `kubernetes` worker. I think we could do this with different groups containing `user-data` with functionality to configure `slurm` versus `kubernetes`, but also retain generic things like `ssh`, `chrony`, `rsyslog`, and lots of other basic things you'd want in any configuration. Taking the previous group list - ssh @@ -319,7 +325,7 @@ where `slurm-client`is something like: name: slurm-client cloud-init: userdata: - write_files: + write_files: - content: | SLURMD_OPTIONS=--conf-server {{ ds.meta_data.groups.slurm-cluster1.slurmctld_server }}:{{ ds.meta_data.groups.slurm-cluster1.slurmctld_port }} path: /etc/sysconfig/slurmd @@ -335,9 +341,9 @@ cloud-init: slurmctld_port: 6817 ``` -and the resulting `user-data` and `meta-data` the cloud-init client gets should include the actions and variables it needs to configure `slurmd` during boot. +and the resulting `user-data` and `meta-data` the cloud-init client gets should include the actions and variables it needs to configure `slurmd` during boot. -If we wanted to move this to a different slurm cluster we could swap it to a `slurm-cluster2` with different settings and the `slurm-client` would still be valid. +If we wanted to move this to a different slurm cluster we could swap it to a `slurm-cluster2` with different settings and the `slurm-client` would still be valid. If we wanted to swap this component to be a kubernetes worker node, all we should have to do is move it from the slurm groups to kubernetes groups: - kube-worker - kube-cluster1 diff --git a/demo/set_cluster_defaults.sh b/demo/set_cluster_defaults.sh index ddb395b07..84879914c 100755 --- a/demo/set_cluster_defaults.sh +++ b/demo/set_cluster_defaults.sh @@ -1,5 +1,9 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + curl -X POST http://localhost:27777/cloud-init/admin/cluster-defaults/ \ -H "Content-Type: application/json" \ -d '{ diff --git a/demo/update_x3003c3b5n0.sh b/demo/update_x3003c3b5n0.sh index 71b27bc77..df8118982 100755 --- a/demo/update_x3003c3b5n0.sh +++ b/demo/update_x3003c3b5n0.sh @@ -1,5 +1,9 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright © 2026 OpenCHAMI a Series of LF Projects, LLC +# +# SPDX-License-Identifier: MIT + # Define the cloud-config content COMPUTE_CLOUD_CONFIG_CONTENT=$(cat <