From 21810aecf7088714a0369002878cd9785e18e0d0 Mon Sep 17 00:00:00 2001 From: praveen Date: Thu, 27 Aug 2026 13:44:29 -0700 Subject: [PATCH] build: parallelize gpuagent build with correct -j dependency graph The build could not use make -j: object compilation had no dependency edge to the third-party libs or generated protos, so -j raced ahead and compiled gpuagent before headers/protos existed. The inner build was also invoked without -j at all, forcing serial object compilation (~hundreds of .cc x 3 variants), which dominated build time. - inner Makefile: declare generated *.pb.cc as grouped-target (&:) outputs so the %.o:%.cc pattern rule can resolve every generated .o under -j; add order-only edges so objects wait for generated *.pb.h and the lib stamp without forcing recompiles on proto refresh. - top Makefile: pass make -j$(GPUAGENT_JOBS) (default nproc) to the inner build; override with GPUAGENT_JOBS=N. - Makefile.lib: .NOTPARALLEL: keeps the six third-party libs serial across each other (each already runs make -j$(nproc) internally; concurrent libs would oversubscribe and OOM grpc/boost). - ci.yml: add step summary (per-binary size/sha256), upload the 4 binaries and build.log as artifacts (retention 10d, if: always()). Clean -j8 build verified successful end-to-end (all 3 gpuagent variants + gpuctl); object+link phase ~449s serial -> ~108s parallel. Co-Authored-By: Claude --- .github/workflows/ci.yml | 73 +++++++++++++++++++++++++++++++++++- Makefile | 4 +- sw/nic/gpuagent/Makefile | 32 +++++++++++----- sw/nic/gpuagent/Makefile.lib | 4 ++ 4 files changed, 102 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbef8209..fb36ed3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,78 @@ jobs: cache-to: type=gha,mode=max - name: Build gpuagent + id: build run: | + # pipefail is on by default in Actions bash, so make's rc survives tee make gpuagent \ GIT_COMMIT=$(git rev-list -1 HEAD --abbrev-commit) \ - BUILD_DATE=$(date +%Y-%m-%dT%H:%M:%S%z) + BUILD_DATE=$(date +%Y-%m-%dT%H:%M:%S%z) \ + 2>&1 | tee build.log + + - name: Build summary + if: always() + run: | + BIN=sw/nic/build/x86_64/sim/bin + { + echo "## gpuagent build" + echo "" + echo "- status: **${{ steps.build.outcome }}**" + echo "- commit: \`$(git rev-list -1 HEAD --abbrev-commit)\`" + echo "" + echo "| binary | size | sha256 |" + echo "|---|---|---|" + for b in gpuagent gpuagent_gim gpuagent_mock gpuctl; do + if [ -f "$BIN/$b" ]; then + sz=$(du -h "$BIN/$b" | cut -f1) + sha=$(sha256sum "$BIN/$b" | cut -c1-12) + echo "| $b | $sz | \`$sha\` |" + else + echo "| $b | — | **MISSING** |" + fi + done + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload gpuagent + if: always() + uses: actions/upload-artifact@v4 + with: + name: gpuagent + path: sw/nic/build/x86_64/sim/bin/gpuagent + retention-days: 10 + if-no-files-found: warn + + - name: Upload gpuagent_gim + if: always() + uses: actions/upload-artifact@v4 + with: + name: gpuagent_gim + path: sw/nic/build/x86_64/sim/bin/gpuagent_gim + retention-days: 10 + if-no-files-found: warn + + - name: Upload gpuagent_mock + if: always() + uses: actions/upload-artifact@v4 + with: + name: gpuagent_mock + path: sw/nic/build/x86_64/sim/bin/gpuagent_mock + retention-days: 10 + if-no-files-found: warn + + - name: Upload gpuctl + if: always() + uses: actions/upload-artifact@v4 + with: + name: gpuctl + path: sw/nic/build/x86_64/sim/bin/gpuctl + retention-days: 10 + if-no-files-found: warn + + - name: Upload build log + if: always() + uses: actions/upload-artifact@v4 + with: + name: build-log + path: build.log + retention-days: 10 + if-no-files-found: warn diff --git a/Makefile b/Makefile index 3b9ecc5d..9a525187 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ CUR_USER:=$(shell whoami) CUR_TIME:=$(shell date +%Y-%m-%d_%H.%M.%S) GPUAGENT_BLD_CONTAINER_IMAGE ?= gpuagent-builder-rhel:9 GPUAGENT_BLD_CONTAINER_IMAGE_UBUNTU ?= gpuagent-bldr-ubuntu:22.04 +# inner build -j width. override: make GPUAGENT_JOBS=N gpuagent. default nproc. +GPUAGENT_JOBS ?= $(shell nproc) CONTAINER_NAME := gpuagent-ctr-${CUR_USER}_${CUR_TIME} CONTAINER_WORKDIR := /usr/src/github.com/ROCm/gpu-agent BUILD_DATE ?= $(shell date +%Y-%m-%dT%H:%M:%S%z) @@ -38,7 +40,7 @@ gpuagent: -v $(CURDIR):$(CONTAINER_WORKDIR) \ -w $(CONTAINER_WORKDIR) \ ${GPUAGENT_BLD_CONTAINER_IMAGE} \ - bash -c " cd $(CONTAINER_WORKDIR) && source ~/.bashrc && make gopkglist && make -C sw/nic/gpuagent all" + bash -c " cd $(CONTAINER_WORKDIR) && source ~/.bashrc && make gopkglist && make -j$(GPUAGENT_JOBS) -C sw/nic/gpuagent all" .PHONY: docker-shell docker-shell: diff --git a/sw/nic/gpuagent/Makefile b/sw/nic/gpuagent/Makefile index a4b99d8c..2b7faff1 100644 --- a/sw/nic/gpuagent/Makefile +++ b/sw/nic/gpuagent/Makefile @@ -69,10 +69,14 @@ SRC := $(shell find $(TOPDIR) -type d \( -path $(EXCLUDE_DIRS) \) -prune -o \ -type d \( -path $(SMI_SRC_DIR) \) -prune -o \ -type d \( -path $(BLD_PROTOGEN_DIR) \) -prune -o \ -type f -name "*.cc" -print) -SRC += $(patsubst $(GPUAGENT_PROTO_DIR)/%.proto, $(GPUAGENT_PROTO_GEN_DIR)/%.pb.cc, $(GPUAGENT_PROTO_SRCS)) -SRC += $(patsubst $(GPUAGENT_PROTO_DIR)/%.proto, $(GPUAGENT_PROTO_GEN_DIR)/%.grpc.pb.cc, $(GPUAGENT_PROTO_SRCS)) -SRC += $(patsubst $(GOGO_PROTO_DIR)/%.proto, $(BLD_PROTOGEN_DIR)/%.pb.cc, $(GOGO_PROTO_SRCS)) -SRC += $(patsubst $(GOGO_PROTO_DIR)/%.proto, $(BLD_PROTOGEN_DIR)/%.grpc.pb.cc, $(GOGO_PROTO_SRCS)) +# generated .cc named so grouped-target rules below can declare them as +# buildable targets; else %.o:%.cc skips them under -j and link fails. +GEN_CPP_PB := $(patsubst $(GPUAGENT_PROTO_DIR)/%.proto, $(GPUAGENT_PROTO_GEN_DIR)/%.pb.cc, $(GPUAGENT_PROTO_SRCS)) +GEN_CPP_GRPC := $(patsubst $(GPUAGENT_PROTO_DIR)/%.proto, $(GPUAGENT_PROTO_GEN_DIR)/%.grpc.pb.cc, $(GPUAGENT_PROTO_SRCS)) +GEN_GOGO_PB := $(patsubst $(GOGO_PROTO_DIR)/%.proto, $(BLD_PROTOGEN_DIR)/%.pb.cc, $(GOGO_PROTO_SRCS)) +GEN_GOGO_GRPC := $(patsubst $(GOGO_PROTO_DIR)/%.proto, $(BLD_PROTOGEN_DIR)/%.grpc.pb.cc, $(GOGO_PROTO_SRCS)) +GEN_CPP_SRCS := $(GEN_CPP_PB) $(GEN_CPP_GRPC) $(GEN_GOGO_PB) $(GEN_GOGO_GRPC) +SRC += $(GEN_CPP_SRCS) SRC_C := $(shell find $(TOPDIR) -type d \( -path $(EXCLUDE_DIRS) \) -prune -o \ -type d \( -path $(EXCLUDE_VENDOR) \) -prune -o \ -type d \( -path $(BLD_DIR) \) -prune -o \ @@ -210,8 +214,14 @@ $(BUILD_LIBS_STAMP): build-libs: $(BUILD_LIBS_STAMP) +# grouped-target (&:, make>=4.3): one protoc call declared producer of all its +# .cc, so %.o:%.cc resolves every generated .o under -j. stamp order-only = protoc. +$(GEN_GOGO_PB) $(GEN_GOGO_GRPC) &: $(GOGO_PROTO_SRCS) | $(BUILD_LIBS_STAMP) + @mkdir -p ${BLD_PROTOGEN_DIR} + LC_ALL=C LD_LIBRARY_PATH=${PROTOC_LIB_PATH} ${PROTOC} --cpp_out=${BLD_PROTOGEN_DIR} --grpc_out=${BLD_PROTOGEN_DIR} ${PROTOC_CPP_OPTS} ${PROTO_INCS} ${PROTOC_DEFS} ${GOGO_PROTO_SRCS} -gen-protos: +# gpuagent protos: gogo dep = shares gogo include path, gogo generated first. +$(GEN_CPP_PB) $(GEN_CPP_GRPC) &: $(GPUAGENT_PROTO_SRCS) $(GEN_GOGO_PB) | $(BUILD_LIBS_STAMP) @mkdir -p ${GPUAGENT_PROTO_GEN_DIR} LC_ALL=C LD_LIBRARY_PATH=${PROTOC_LIB_PATH} ${PROTOC} --cpp_out=${GPUAGENT_PROTO_GEN_DIR} --grpc_out=${GPUAGENT_PROTO_GEN_DIR} ${PROTOC_CPP_OPTS} ${PROTO_INCS} ${PROTOC_DEFS} ${GPUAGENT_PROTO_SRCS} @mkdir -p ${GPUAGENT_PROTO_GO_GEN_DIR} @@ -220,14 +230,18 @@ gen-protos: --gogofast_out=Mgogo.proto=github.com/gogo/protobuf/gogoproto,plugins=grpc:${GPUAGENT_PROTO_GO_GEN_DIR} \ types.proto ${GPU_PROTO_GO_FILES} +# phony aliases -> real outputs (link targets + `make gen-protos` still work). +.PHONY: gen-protos gogo-protos +gogo-protos: $(GEN_GOGO_PB) $(GEN_GOGO_GRPC) +gen-protos: $(GEN_CPP_PB) $(GEN_CPP_GRPC) + +# order-only: objects need generated *.pb.h present, no recompile on proto refresh. +$(OBJ) $(OBJ_GIM) $(OBJ_MOCK): | $(BUILD_LIBS_STAMP) $(GEN_CPP_SRCS) + gpuctl: @echo "building gpuctl" CGO_ENABLED=0 go build -C cli -ldflags="-s -w -X github.com/ROCm/gpu-agent/sw/nic/gpuagent/cli/cmd.GpuctlVersion=$(GPUAGENT_VERSION)" -o ${BLD_BIN_DIR}/gpuctl -gogo-protos: - @mkdir -p ${BLD_PROTOGEN_DIR} - LC_ALL=C LD_LIBRARY_PATH=${PROTOC_LIB_PATH} ${PROTOC} --cpp_out=${BLD_PROTOGEN_DIR} --grpc_out=${BLD_PROTOGEN_DIR} ${PROTOC_CPP_OPTS} ${PROTO_INCS} ${PROTOC_DEFS} ${GOGO_PROTO_SRCS} - $(OBJ_DIR)/%.o: $(TOPDIR)/%.cc @mkdir -p $(dir $@) # Create the necessary subdirectories in temp $(CC) $(CFLAGS) $(DEFS) $(INCS_AMD_SMI) -DAMD_SMI -D__FNAME__=__FILE__ -DGPUAGENT_VERSION=\"$(GPUAGENT_VERSION)\" -c $< -o $@ diff --git a/sw/nic/gpuagent/Makefile.lib b/sw/nic/gpuagent/Makefile.lib index 930586de..4fed375e 100644 --- a/sw/nic/gpuagent/Makefile.lib +++ b/sw/nic/gpuagent/Makefile.lib @@ -17,6 +17,10 @@ CC := gcc CXX := g++ +# libs serial across each other: each recipe already runs make -j$(nproc) +# internally; concurrent libs = Nlibs x nproc oversubscribe -> OOM grpc/boost. +.NOTPARALLEL: + # Third-party build targets .PHONY: third-party-libs protobuf-build abseil-build grpc-build zeromq-build libev-build boost-build