From 449c6f13cc6a50fe1dd37668dd85e97717b0b55a Mon Sep 17 00:00:00 2001 From: Miguel Lozano Date: Sun, 8 Nov 2026 23:50:09 -0500 Subject: [PATCH] refactor(docs): align public documentation metrics and update caddy pins - Update caddy_ui_image to v1.4.2 with verified multi-arch sha256 digest - Update caddy_image to v3.6.4 with verified multi-arch sha256 digest - Purge 'Nuntu' codename and unbaselined metrics from public documentation - Unify Vault-backed secrets to SOPS+age in features - Align Production Ready status with active inventory and Molecule test suite - Unify application profiles catalog count across README sections --- README.md | 33 +++++++++++++---------------- docs/finops/FINOPS-BASELINE.md | 24 ++++++++++----------- docs/operations/VERSION_PINS.md | 2 +- inventory/group_vars/all/images.yml | 12 +++++------ 4 files changed, 33 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 2d52075..9d4f50f 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ _A layered infrastructure stack built with Ansible, Docker and Zero Trust princi - [Overview](#overview) - [Features](#features) - [Value Snapshot for CTO/CFO](#value-snapshot-for-ctocfo) -- [FinOps Case Study (Nuntu)](#finops-case-study-nuntu) +- [FinOps Field Case Study](#finops-field-case-study) - [Field Validation Evidence](#field-validation-evidence) - [Compliance & Standards](#compliance--standards) - [Quick Start](#quick-start) @@ -65,10 +65,10 @@ The project aligns to **NIST 800-53** controls and emphasizes secrets handling, ## Features -- 🛡️ **Security-first hardening** with SSH restrictions, firewall controls, audit logging, CrowdSec integration, and Vault-backed secrets. +- 🛡️ **Security-first hardening** with SSH restrictions, firewall controls, audit logging, CrowdSec integration, and SOPS + age encrypted secrets. - 🔐 **Zero-trust networking** through Tailscale ACL-driven access and minimized public exposure. - 🧱 **Modular architecture** using Ansible roles for security, Docker orchestration, observability, compliance, ingress, and Portainer edge operations. -- 📦 **Optional application bundles** for Chatwoot, Metabase, n8n, OpenWebUI, Twenty CRM, and Uptime Kuma. +- 📦 **Optional application bundles** for Chatwoot, Metabase, n8n, NocoDB, OpenWebUI, Twenty CRM, Uptime Kuma, and FastAPI (community template). - 📈 **Operational visibility** with exporter and observability stack support when capacity allows. - 🧪 **Validation gates** executed through Make targets (uv-backed) for linting, syntax checks, and secret scanning. - 🧭 **Documented operating model** with commands centralized in `docs/operations/OPERATIONS_RUNBOOK.md`. @@ -87,9 +87,9 @@ In less than 30 seconds: --- -## FinOps Case Study (Nuntu) +## FinOps Field Case Study -This project pattern has been applied to a real-world migration scenario (codenamed **Nuntu**) focused on sovereignty, security, and OpEx reduction. +This project pattern has been applied to a real-world sovereign cloud-exit migration scenario focused on data sovereignty, security, and OpEx reduction. ### Business problem @@ -106,14 +106,12 @@ This project pattern has been applied to a real-world migration scenario (codena ### Measured outcomes (reported case) -| Metric | Before (SaaS Sprawl) | After (Sovereign Self-Hosted) | Impact | -| ---------------------- | -------------------- | ------------------------------------ | -------------------------- | -| Annual Software OpEx | $X (baseline) | $0.3X | **-70%** | -| Platform Uptime (Prod) | ~99.5% | 99.8% | Higher reliability | -| Data Sovereignty | 0% | 100% | Full control | -| WAF Efficacy | N/A | >99% block rate, <1% false positives | Enterprise-grade perimeter | -| Incident Response Time | Hours/Days | Minutes | Stronger resilience | -| Vendor Risk | Critical | Negligible | Supply-chain risk reduced | +| Metric | Before (SaaS Sprawl) | After (Sovereign Self-Hosted) | Impact | +| ---------------------- | -------------------- | ----------------------------- | ------------------------- | +| Annual Software OpEx | $X (baseline) | $0.3X | **-70%** | +| Platform Uptime (Prod) | ~99.5% | 99.8% | Higher reliability | +| Data Sovereignty | 0% | 100% | Full control | +| Vendor Risk | Critical | Negligible | Supply-chain risk reduced | > **Scope note:** This case study is a field implementation narrative and not a vendor benchmark report. Results depend on workload, architecture, and governance discipline. @@ -121,12 +119,11 @@ This project pattern has been applied to a real-world migration scenario (codena ## Field Validation Evidence -Current documented execution evidence in this repository corresponds to a lab/staging scope with: +The platform architecture and hardening suite are **Production Ready**, validated across active production operations and staged topologies: -- **1 Brain node** -- **2 Muscle nodes** - -This is intentionally presented as operational proof-of-execution while broader fleet rollouts are scheduled. +- **Production Reference Fleet**: 1 Management Node (`brain-1`) + 1 Compute Worker Node (`muscle-1`) in active operation. +- **Multi-Node Staging & Lab Validation**: Multi-node cluster topology validated across 1 Brain + 2 Muscle nodes. +- **Automated Verification**: Multi-layer Molecule verification suite (`L1_os_baseline`, `L2_compliance`, `L3_observability`, `L4_networking`, `backup`) tested on Debian 12 (`bookworm`) and Ubuntu 22.04 (`jammy`). ### Deployment proof screenshots diff --git a/docs/finops/FINOPS-BASELINE.md b/docs/finops/FINOPS-BASELINE.md index df8b227..63dac39 100644 --- a/docs/finops/FINOPS-BASELINE.md +++ b/docs/finops/FINOPS-BASELINE.md @@ -16,7 +16,7 @@ repo: github.com/Developmi/hardened This document tracks cost estimation and financial governance across the Developmi Hardened. It establishes a per-node cost baseline for OpenSource-scoped layers (L1–L6), documents the -boundary with private cloud provisioning costs (L0), and references the Nuntu field case study +boundary with private cloud provisioning costs (L0), and references the sovereign migration field case study for aggregate OpEx comparison. **Scope rule:** L0 cloud provisioning costs are private and explicitly excluded from this @@ -78,16 +78,14 @@ and incur no platform cost. ## Case Study Summary -> Source: [README.md](../../README.md) - FinOps Case Study (Nuntu) +> Source: [README.md](../../README.md) - FinOps Field Case Study -| Metric | Before (SaaS Sprawl) | After (Sovereign Self-Hosted) | Impact | -| ---------------------- | -------------------- | ------------------------------ | -------------------------- | -| Annual Software OpEx | $X (baseline) | $0.3X | **-70%** | -| Platform Uptime (Prod) | ~99.5% | 99.8% | Higher reliability | -| Data Sovereignty | 0% | 100% | Full control | -| WAF Efficacy | N/A | >99% block, <1% false positive | Enterprise-grade perimeter | -| Incident Response Time | Hours/Days | Minutes | Stronger resilience | -| Vendor Risk | Critical | Negligible | Supply-chain risk reduced | +| Metric | Before (SaaS Sprawl) | After (Sovereign Self-Hosted) | Impact | +| ---------------------- | -------------------- | ----------------------------- | ------------------------- | +| Annual Software OpEx | $X (baseline) | $0.3X | **-70%** | +| Platform Uptime (Prod) | ~99.5% | 99.8% | Higher reliability | +| Data Sovereignty | 0% | 100% | Full control | +| Vendor Risk | Critical | Negligible | Supply-chain risk reduced | This is a field implementation narrative, not a vendor benchmark. Results depend on workload, architecture, and governance discipline. @@ -102,7 +100,7 @@ Update this document when any of the following occur: 2. **Provider change** shifts the VPS-equivalent pricing tiers (e.g., ARM64 Ampere nodes change the cost floor). 3. **Significant price change** in the self-hosted stack components (Docker, Caddy, PostgreSQL, etc.). 4. **New layer** or role added that materially affects compute, storage, or network cost. -5. **Case study refresh** - new field data from additional migrations or expanded Nuntu fleet. +5. **Case study refresh** - new field data from additional migrations or expanded fleet. To update: @@ -114,13 +112,13 @@ To update: | Date | Change | | ---------- | -------------------------------------------------------------------------------- | -| 2026-07-05 | Initial baseline: per-node table, L0/L1-L6 separation, Nuntu case study summary. | +| 2026-07-05 | Initial baseline: per-node table, L0/L1-L6 separation, field case study summary. | --- ## Cross-References -- [README.md](../../README.md) - FinOps Case Study (Nuntu) with aggregate before/after metrics +- [README.md](../../README.md) - FinOps Field Case Study with aggregate before/after metrics - [ARCHITECTURE.md](../architecture/ARCHITECTURE.md) - §5 L0/L1–L6 scope separation and trust boundaries; §7 host classes (brain/muscle/local) - [ROADMAP.md](../project/ROADMAP.md) - FinOps Specialist role - [ARCHITECTURE.md §5](../architecture/ARCHITECTURE.md#5-7-layer-architecture-model-full-specification) - Layer model (L0-L6) reference. diff --git a/docs/operations/VERSION_PINS.md b/docs/operations/VERSION_PINS.md index 9906a5f..2da18c9 100644 --- a/docs/operations/VERSION_PINS.md +++ b/docs/operations/VERSION_PINS.md @@ -28,7 +28,7 @@ machine" failures, and aligns with the **"Boring Technology"** principle. | 3 | **ansible-lint** | `==26.6.0` | Ensures playbooks follow the same rule set on every machine. Prevents CI-vs-local drift. Defined in `pyproject.toml`. The version is aligned with the ansible-core pin - upgrading one typically requires the other. | Check [ansible-lint changelog](https://github.com/ansible/ansible-lint/releases) for new rules. Run `make lint` and fix violations before committing the bump. | | 4 | **Docker Engine** | Per-OS pins in `inventory/group_vars/all/images.yml` | Reproducible cluster behaviour depends on identical Engine versions across nodes. Pins are per-distribution-codename to match Docker's packaging convention (e.g., `5:29.5.3-1~debian.12~bookworm` for Debian 12). | Test on a single node first: `make deploy-tags PLAYBOOK=playbooks/l6/engine.yml ANSIBLE_TAGS='docker' --limit=`. Verify `docker version` matches expected output. Roll back by reverting the pin in `images.yml` and re-running the playbook. | | 5 | **Ansible Galaxy Collections** | Pinned in `requirements.yml` | Four collections are required: `community.general` (13.0.1), `ansible.posix` (2.2.0), `community.docker` (5.2.1), `community.sops` (2.4.0). The sops collection provides the `community.sops.sops` vars plugin for transparent SOPS+age secret loading. Pins prevent Galaxy from pulling breaking major versions on a fresh `ansible-galaxy install`. | Run `make install-collections` in a clean environment. Execute `make dry-run` for all playbooks. Check each collection's changelog for module deprecations. | -| 6 | **Caddy WAF + UI (reverse proxy)** | `ghcr.io/developmi/caddy-waf:v3.6.1` (+ digest `sha256:d6a824c8…`) and `ghcr.io/developmi/caddy-waf-ui:v1.4.0` (+ digest `sha256:a0a04b7b…`) in `images.yml`, verified by the L4 role | The ingress layer is the first line of defence. Pinning the WAF image ensures consistent Coraza rule sets and TLS handling across deployments. Defined in `inventory/group_vars/all/images.yml`. v3.6.1, coraza-caddy 2.6.1, coraza 3.7.0, OWASP CRS 4.30.0; UI v1.4.0. The digest gate is enforced at deploy time. | Build and push the new image tag. Verify WAF rule set compatibility (Coraza version must match). Run `make deploy-tags PLAYBOOK=playbooks/l4/edge.yml ANSIBLE_TAGS='l4-networking'`. Smoke-test a health endpoint through the proxy. | +| 6 | **Caddy WAF + UI (reverse proxy)** | `ghcr.io/developmi/caddy-waf:v3.6.4` (+ digest `sha256:7233e89e…`) and `ghcr.io/developmi/caddy-waf-ui:v1.4.2` (+ digest `sha256:eee50db5…`) in `images.yml`, verified by the L4 role | The ingress layer is the first line of defence. Pinning the WAF image ensures consistent Coraza rule sets and TLS handling across deployments. Defined in `inventory/group_vars/all/images.yml`. v3.6.4, coraza-caddy 2.6.1, coraza 3.7.0, OWASP CRS 4.30.0; UI v1.4.2. The digest gate is enforced at deploy time. | Build and push the new image tag. Verify WAF rule set compatibility (Coraza version must match). Run `make deploy-tags PLAYBOOK=playbooks/l4/edge.yml ANSIBLE_TAGS='l4-networking'`. Smoke-test a health endpoint through the proxy. | | 7 | **Portainer (optional manager)** | Server `2.39.5`, Agent `2.39.5` | Manager and agent MUST stay at the same version. Portainer is optional (ADR-07) but when deployed, version drift between server and agent breaks edge-agent communication. Defined in `inventory/group_vars/all/managers/portainer.yml`. | Review [Portainer release notes](https://docs.portainer.io/v/ce-2.39/release-notes). Upgrade agent first, then server. Verify edge-agent connectivity in the Portainer UI before considering the upgrade complete. | | 8 | **Observability Stack** | VictoriaMetrics `v1.148.0`, Loki `3.6.13`, Grafana `13.1.1`, Node Exporter `v1.12.1`, cAdvisor `v0.60.5`, Grafana Alloy `v1.18.1`, blackbox-exporter `v0.27.0` | Monitoring is a contractual interface (L3). Version drift between exporters, databases, and dashboards causes silent metric gaps. All images pinned in `inventory/group_vars/all/images.yml`. Alloy (successor of promtail, EOL 2026-03-02) ships Caddy log files to Loki. blackbox-exporter stays on `v0.27.0`: `v0.28.0` (2025-12-04) ships with known CRITICAL CVEs in the official image (CVE-2025-68121 Go stdlib, CVE-2026-33186 grpc — prometheus/blackbox_exporter#1567, no rebuild released); revisit on a patched release. | Upgrade in this order: exporters → VictoriaMetrics/Loki → Grafana. After each step, verify dashboards render and alert rules still fire. Roll back by reverting the tag and re-deploying the monitoring playbook. | | 9 | **Application Images** | Per-app in compose pin `${APP_IMAGE:-org/image:tag}` (`apps//docker-compose.yml`) + mirrored `_IMAGE` in `apps//.env.example` | All 11 apps pinned to registry-verified latest stable (2026-07-31) - see the [application pin table](#application-image-pins-registry-verified-2026-07-31) below. App images are the most volatile - upstream releases may break DB migrations or change env-var contracts. | Check the upstream changelog for breaking changes (especially DB migration notes). Test on a staging environment. Run the app's health endpoint after deploy. Keep the previous tag in a comment for one release cycle as quick rollback. | diff --git a/inventory/group_vars/all/images.yml b/inventory/group_vars/all/images.yml index eabd797..7a67df7 100644 --- a/inventory/group_vars/all/images.yml +++ b/inventory/group_vars/all/images.yml @@ -10,21 +10,21 @@ # ----------------------------------------------------------------------------- # Custom image namespace image_namespace: "{{ vault_image_namespace }}" # AC-3: value from secrets.sops.yml only -# Caddy image (SSOT: caddy-waf repo releases; v3.6.2) -caddy_image: "ghcr.io/developmi/caddy-waf:v3.6.2" +# Caddy image (SSOT: caddy-waf repo releases; v3.6.4) +caddy_image: "ghcr.io/developmi/caddy-waf:v3.6.4" # SHA256 digest (OCI index, multi-arch amd64+arm64) for supply chain verification. # Enforced as a verification gate by the L4 role (docker_image_info digest match, # tasks/caddy.yml 'Verify Caddy image digest if specified'). -caddy_image_digest: "sha256:572968506168150bed96e293c9461578d911a980ecb8ef91d6672f381dda2cbb" +caddy_image_digest: "sha256:7233e89ea232e0b6a89013f2f48c92f0d1342ba8629c62c1f455dfeac0e947ce" -# Caddy WAF UI sidecar image (SSOT: caddy-waf-ui repo releases; v1.4.0) -caddy_ui_image: "ghcr.io/developmi/caddy-waf-ui:v1.4.0" +# Caddy WAF UI sidecar image (SSOT: caddy-waf-ui repo releases; v1.4.2) +caddy_ui_image: "ghcr.io/developmi/caddy-waf-ui:v1.4.2" # SHA256 digest (OCI index, multi-arch amd64+arm64) for supply chain verification. # Enforced as a verification gate by the L4 role (docker_image_info digest match, # tasks/caddy.yml 'Verify UI image digest if specified'). -caddy_ui_image_digest: "sha256:a0a04b7b60284d3a3a3b5e5b1692c6e20c734256699a25360c49ec9d930c5b05" +caddy_ui_image_digest: "sha256:eee50db52617328f42cf1eb2d304f320b163e7fca2e6fecb4e6678adaf63ec8c" # ----------------------------------------------------------------------------- # GITHUB CONTAINER REGISTRY