From b450c9015a43e5e77bfb1bd1a8d6b9954f1f0de6 Mon Sep 17 00:00:00 2001 From: Miguel Lozano Date: Wed, 4 Nov 2026 14:28:25 -0500 Subject: [PATCH 1/2] refactor(ops): purge local device hardening playbook and Pop!_OS conditionals --- Makefile | 4 +- inventory/group_vars/all/main.yml | 8 +- .../group_vars/all/secrets.sops.yml.example | 5 +- inventory/group_vars/local/main.yml | 43 -- inventory/hosts.ini.example | 7 - playbooks/ops/local-devices.yml | 596 ------------------ .../general/defaults/tailscale.yml | 2 +- .../general/tasks/security/nftables_rules.yml | 6 - .../general/tasks/tailscale/main.yml | 12 +- .../tailscale/tailscale-acls.json.j2 | 25 +- roles/L6_runtime/general/tasks/main.yml | 11 +- scripts/setup.sh | 1 - 12 files changed, 16 insertions(+), 704 deletions(-) delete mode 100644 inventory/group_vars/local/main.yml delete mode 100644 playbooks/ops/local-devices.yml diff --git a/Makefile b/Makefile index 62096ee..198d3dc 100644 --- a/Makefile +++ b/Makefile @@ -291,8 +291,6 @@ backup-now: $(ANSIBLE_RUN) playbooks/l6/backup-stack.yml $(ANSIBLE_FLAGS) --tags stack_backup --limit brain provision-host: $(ANSIBLE_RUN) playbooks/ops/bootstrap.yml $(ANSIBLE_FLAGS) -deploy-local: - $(ANSIBLE_RUN) playbooks/ops/local-devices.yml $(ANSIBLE_FLAGS) verify-lockdown: @$(MAKE) verify COMPONENT=lockdown verify-tailscale: @@ -318,6 +316,6 @@ test-layer: install-collections check-toolchain setup-toolchain deploy-l1 deploy-lockdown \ deploy-compliance reconnect-tailscale validate-l1 validate-l2 audit-full \ deploy-exporters deploy-monitoring-stack verify-caddy heal-caddy \ - deploy-backup-stack deploy-backup-engine backup-now provision-host deploy-local \ + deploy-backup-stack deploy-backup-engine backup-now provision-host \ verify-lockdown verify-tailscale verify-crowdsec verify-auditd verify-observability \ verify-timers test-layer diff --git a/inventory/group_vars/all/main.yml b/inventory/group_vars/all/main.yml index 5c71a06..3d6cecd 100644 --- a/inventory/group_vars/all/main.yml +++ b/inventory/group_vars/all/main.yml @@ -83,18 +83,12 @@ systemd_nproc_hard: 65536 # {brain,muscle,local} > all > role defaults. # Every tunable below has a default - override at the appropriate layer. -# ─── OS Detection & Normalization ──────────────────────────────────────────── -# _os_family normalizes Pop OS → Ubuntu before any L2+ roles execute. -# Used by downstream dispatchers (firewall, packages, etc.). -_os_family: "{{ 'Ubuntu' if ansible_facts['distribution'] in ['Pop', 'Pop!_OS'] else ansible_facts['os_family'] }}" - # os_firewall_backend (NIST SC-7 / CIS §3.5 Track A): # Unified in-kernel nftables across all Linux distributions (Debian, Ubuntu). os_firewall_backend: "nftables" # ─── Hardening Profile Dispatcher ──────────────────────────────────────────── -# server: strict firewalls, short fail2ban bans, CrowdSec enforcement -# workstation: lenient firewalls, longer fail2ban bans, CrowdSec detection_only +# Server-grade hardening: strict firewalls, fail2ban bans, CrowdSec enforcement hardening_profile: server # ─── Network & Access ──────────────────────────────────────────────────────── diff --git a/inventory/group_vars/all/secrets.sops.yml.example b/inventory/group_vars/all/secrets.sops.yml.example index ffce842..48eae8c 100644 --- a/inventory/group_vars/all/secrets.sops.yml.example +++ b/inventory/group_vars/all/secrets.sops.yml.example @@ -44,7 +44,7 @@ # ============================================================================= # Tailnet name, e.g. "mycorp.ts.net". Used to build the ACL API URL. -# Used by playbooks/l2/hardening.yml:103 and playbooks/ops/local-devices.yml:60. +# Used by playbooks/l2/hardening.yml:103. tailscale_tailnet: "tailnet.example.invalid" # Optional ACL groups: map of group name -> member list. @@ -252,8 +252,7 @@ vault_image_namespace: "ghcr.io/example-org" vault_github_username: "example-github-user" # GitHub Personal Access Token (read:packages) used to log in to the registry. -# Used by roles/L4_networking/caddy/tasks/caddy.yml:377 (docker login) and -# validated in playbooks/ops/local-devices.yml:38. +# Used by roles/L4_networking/caddy/tasks/caddy.yml:377 (docker login). # Required for L4/L6 deployments. vault_github_token: "REPLACE_WITH_GITHUB_PAT" diff --git a/inventory/group_vars/local/main.yml b/inventory/group_vars/local/main.yml deleted file mode 100644 index 81280ad..0000000 --- a/inventory/group_vars/local/main.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -# ============================================================================= -# LOCAL DEVICE - WORKSTATION CONFIGURATION -# ============================================================================= -# Personal devices (desktops, laptops) with manual Tailscale auth. -# Uses workstation-appropriate hardening - less aggressive than servers. -# ============================================================================= - -# Device role identification -server_role: local -server_type: workstation -host_class: local - -# L1-L2 hardening profile - workstation-grade (lenient fail2ban, detection-only CrowdSec) -crowdsec_mode: detection_only -hardening_profile: workstation - -# Security profile (workstation-appropriate) -# Fail2ban: shorter bantime for laptops (avoid lockout without IPMI/console access) -fail2ban_bantime: 10m -fail2ban_maxretry: 5 - -# Disable server-only features -enable_observability_stack: false -enable_metrics_exporters: false -enable_backups: false - -# Tailscale auth is manual on local workstations - skip auth key validation -tailscale_manual_auth: true - -# No cloud provider (personal hardware) -cloud_provider: none - -# ============================================================================= -# Supported OS distributions for local devices: -# - Ubuntu 22.04 / 24.04 (x86_64 + ARM64) -# - Debian 11 / 12 (x86_64 + ARM64) -# - Pop OS 22.04 (Ubuntu-based - uses same apt repos and packages) -# -# Pop OS reports ansible_facts['distribution'] as 'Pop' or 'Pop!_OS'. -# The playbook normalizes this to 'Ubuntu' for docker/tailscale repo lookups. -# All apt packages (UFW, fail2ban, etc.) are fully compatible. -# ============================================================================= diff --git a/inventory/hosts.ini.example b/inventory/hosts.ini.example index 91937c1..9244073 100644 --- a/inventory/hosts.ini.example +++ b/inventory/hosts.ini.example @@ -31,13 +31,6 @@ brain-1 ansible_host=YOUR_BRAIN_PUBLIC_IP public_ip=YOUR_BRAIN_PUBLIC_IP node_ro muscle-1 ansible_host=YOUR_MUSCLE_PUBLIC_IP public_ip=YOUR_MUSCLE_PUBLIC_IP node_role=worker caddy_node_id=muscle-1 # muscle-2 ansible_host=ANOTHER_PUBLIC_IP public_ip=ANOTHER_PUBLIC_IP node_role=worker caddy_node_id=muscle-2 -# ===== LOCAL DEVICES ===== -# Personal machines connected through Tailscale. -# Only hardening + Docker are installed (no application stack). -# ansible_user is REQUIRED per host entry (see identity convention above). -[local] -# local-pc ansible_host=100.x.x.x - # ===== FUNCTIONAL GROUPS ===== [observability:children] brain diff --git a/playbooks/ops/local-devices.yml b/playbooks/ops/local-devices.yml deleted file mode 100644 index 7124b17..0000000 --- a/playbooks/ops/local-devices.yml +++ /dev/null @@ -1,596 +0,0 @@ ---- -# ============================================================================== -# Developmi Hardened | Ops - Local Device Hardening -# ============================================================================== -# LOCAL DEVICE HARDENING - Personal Workstations & Laptops -# ============================================================================== -# Purpose: Apply security hardening to personal devices over Tailscale mesh. -# Connectivity: Tailscale 100.x.x.x IPs only (manual auth prerequisite). -# Prerequisites: -# 1. Tailscale installed and logged in: tailscale up --operator=$USER -# 2. Device added to [local] in inventory/hosts.ini with Tailscale IP -# 3. No public_ip needed - devices connect directly via 100.x IPs -# -# Run: -# make deploy-local -# ansible-playbook -i inventory/hosts.ini playbooks/ops/local-devices.yml --ask-vault-pass -# Behavior: -# - ACL is applied FIRST from localhost (enables SSH before device contact) -# - Offline devices are silently skipped (ignore_unreachable: true) -# - Tailscale auth is NOT automated (manual only for local devices) -# - CrowdSec intrusion prevention is skipped (not applicable to workstations) -# - Server playbooks (site.yml, stacks.yml) are completely untouched -# ============================================================================== - -# ============================================================================== -# PHASE 0: PRE-FLIGHT VALIDATION -# ============================================================================== -- name: "PHASE 0: Pre-flight Validation" - hosts: localhost - gather_facts: true - become: false - tags: [always] - tasks: - - name: Validate SOPS variables - ansible.builtin.assert: - that: - - vault_github_token is defined - - vault_github_token | length > 0 - - github_username is defined - - github_username | length > 0 - fail_msg: "❌ SOPS secrets missing. Run: make sops-edit" - success_msg: "✅ SOPS variables validated" - - - name: Validate Tailscale subnet is configured - ansible.builtin.assert: - that: - - tailscale_subnet is defined - - tailscale_subnet | length > 0 - fail_msg: "❌ tailscale_subnet must be defined in group_vars/all/main.yml" - - - name: Validate Tailscale ACL credentials - ansible.builtin.assert: - that: - - tailscale_acl_key is defined - - tailscale_acl_key | trim | length > 20 - - tailscale_acl_client_id is defined - - tailscale_acl_client_id | trim | length > 5 - - tailscale_tailnet is defined - - tailscale_tailnet | trim | length > 0 - fail_msg: "❌ Tailscale ACL OAuth credentials missing in vault (tailscale_acl_key, tailscale_acl_client_id, tailscale_tailnet)" - - - name: Display pre-flight summary - ansible.builtin.debug: - msg: - - "=========================================" - - "✅ PRE-FLIGHT VALIDATION PASSED" - - "=========================================" - - "Target group: [local] ({{ groups['local'] | default([]) | length }} device(s))" - - "Tailscale subnet: {{ tailscale_subnet }}" - - "Tailscale tailnet: {{ tailscale_tailnet }}" - - "Manual Tailscale auth required for each device" - - "=========================================" - -# ============================================================================== -# PHASE 0.1: ACL PRE-CONFIGURATION (applied before SSH to local devices) -# ============================================================================== -- name: "PHASE 0.1: Tailscale ACL Pre-configuration" - hosts: localhost - # Controller-side play: ACL must be applied BEFORE SSH to local devices. - # connection: local + become: false as PLAY keywords (never vars) make sudo - # on the controller impossible, mirroring playbooks/ops/bootstrap.yml. - connection: local - gather_facts: false - become: false - tags: [tailscale, acl, security, always] - tasks: - - name: Render ACL policy with local device rules - ansible.builtin.set_fact: - tailscale_acl_policy: "{{ lookup('template', playbook_dir + '/../../roles/L2_compliance/general/templates/tailscale/tailscale-acls.json.j2') | from_json }}" - - - name: Exchange ACL OAuth client for short-lived API token - ansible.builtin.uri: - url: https://api.tailscale.com/api/v2/oauth/token - method: POST - body_format: form-urlencoded - body: - client_id: "{{ tailscale_acl_client_id | trim }}" - client_secret: "{{ tailscale_acl_key | trim }}" - grant_type: client_credentials - status_code: 200 - return_content: true - register: tailscale_acl_oauth_token - no_log: true - - - name: Store short-lived ACL API token - ansible.builtin.set_fact: - tailscale_acl_bearer_token: "{{ tailscale_acl_oauth_token.json.access_token }}" - no_log: true - - - name: Validate ACL API bearer token - ansible.builtin.assert: - that: - - tailscale_acl_bearer_token is defined - - tailscale_acl_bearer_token | trim | length > 20 - fail_msg: "Unable to obtain a valid Tailscale API bearer token for ACL management." - no_log: true - - - name: Validate Tailscale ACL policy with API - ansible.builtin.uri: - url: "https://api.tailscale.com/api/v2/tailnet/{{ tailscale_tailnet | trim }}/acl/validate" - method: POST - headers: - Authorization: "Bearer {{ tailscale_acl_bearer_token }}" - Content-Type: "application/json" - body_format: json - body: "{{ tailscale_acl_policy }}" - status_code: 200 - no_log: true - - - name: Apply Tailscale ACLs via API - ansible.builtin.uri: - url: "https://api.tailscale.com/api/v2/tailnet/{{ tailscale_tailnet | trim }}/acl" - method: POST - headers: - Authorization: "Bearer {{ tailscale_acl_bearer_token }}" - Content-Type: "application/json" - body_format: json - body: "{{ tailscale_acl_policy }}" - status_code: [200, 201] - no_log: true - - - name: Display ACL configuration result - ansible.builtin.debug: - msg: - - "✅ Tailscale ACL applied with local device rules" - - " tag:local registered in tagOwners" - - " admin → tag:local:22 (operator SSH access)" - - " tag:local → tag:local:22 (inter-device SSH)" - -# ============================================================================== -# PHASE 0.15: SSH CONNECTIVITY VALIDATION -# ============================================================================== -- name: "PHASE 0.15: SSH Connectivity Validation" - hosts: local - gather_facts: false - become: false - ignore_unreachable: true - any_errors_fatal: false - tags: [always] - tasks: - - name: "Test SSH connectivity to device" - ansible.builtin.wait_for_connection: - connect_timeout: 10 - timeout: 10 - register: ssh_connectivity - ignore_errors: true - ignore_unreachable: true - - - name: "Diagnose unreachable device" - ansible.builtin.debug: - msg: - - "======================================================================" - - " WARNING: {{ inventory_hostname }} is UNREACHABLE via SSH on port 22" - - "======================================================================" - - "" - - " Possible causes:" - - " 1. UFW firewall is blocking port 22 on the device" - - " 2. SSH service (sshd) is not running or not installed" - - " 3. Tailscale is not connected or not running" - - " 4. Device is powered off or not on the same network" - - "" - - " Manual recovery steps:" - - " → SSH to the device directly (console or local terminal)" - - " → Run: sudo ufw allow from 100.64.0.0/10 to any port 22 proto tcp" - - " → Run: sudo systemctl enable --now ssh" - - " → Run: tailscale status (verify connected to tailnet)" - - "" - - " After fixing, re-run this playbook:" - - " ansible-playbook -i inventory/hosts.ini playbooks/ops/local-devices.yml --ask-vault-pass" - - "======================================================================" - when: ssh_connectivity is unreachable or ssh_connectivity.failed | default(false) - - - name: "Device is reachable - proceed" - ansible.builtin.debug: - msg: " {{ inventory_hostname }} is reachable via SSH on port 22" - when: ssh_connectivity is not unreachable and not (ssh_connectivity.failed | default(false)) - -# ============================================================================== -# PHASE 0.2: APT LOCK PRE-FLIGHT -# ============================================================================== -- name: "PHASE 0.2: APT Lock Pre-flight" - hosts: local - become: true - gather_facts: false - ignore_unreachable: true - any_errors_fatal: false - tags: [always] - pre_tasks: - - name: "Pre-flight: Stop APT systemd timers" - ansible.builtin.systemd: - name: "{{ item }}" - state: stopped - enabled: false - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - failed_when: false - - - name: "Pre-flight: Track timer state for restore" - ansible.builtin.set_fact: - apt_timers_stopped: true - cacheable: false - - - name: "Pre-flight: Detect APT lock holders" - ansible.builtin.shell: | - fuser /var/lib/apt/lists/lock /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/cache/apt/archives/lock 2>/dev/null - register: apt_lock_holders - changed_when: false - failed_when: false - - - name: "Pre-flight: Enumerate running apt/dpkg processes" - ansible.builtin.shell: | - pgrep -a '^(apt|apt-get|dpkg)$' - register: apt_processes - changed_when: false - failed_when: false - - - name: "Pre-flight: Inspect lock holder process age" - ansible.builtin.shell: | - for pid in $(echo "{{ apt_lock_holders.stdout | default('') }}" | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -u); do - etimes=$(ps -o etimes= -p $pid 2>/dev/null || echo "unknown") - cmd=$(ps -o args= -p $pid 2>/dev/null || echo "unknown") - echo "PID=$pid ELAPSED=${etimes}s CMD='${cmd}'" - done - register: apt_lock_process_info - changed_when: false - failed_when: false - when: apt_lock_holders.stdout | default('') != "" - - - name: "Pre-flight: Auto-kill stale apt processes or force-kill" - ansible.builtin.shell: | - force="{{ apt_force_cleanup | default(false) | bool }}" - killed=false - for pid in $(echo "{{ apt_lock_holders.stdout | default('') }}" | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -u); do - etimes=$(ps -o etimes= -p $pid 2>/dev/null || echo "0") - cmd=$(ps -o args= -p $pid 2>/dev/null || echo "unknown") - if [ "$force" = "True" ]; then - kill -9 $pid 2>/dev/null && echo "KILLED (forced) PID=$pid CMD='${cmd}'" && killed=true - elif [ "$etimes" -gt 900 ] 2>/dev/null; then - kill -9 $pid 2>/dev/null && echo "KILLED (stale, ${etimes}s) PID=$pid CMD='${cmd}'" && killed=true - fi - done - if [ "$killed" = "true" ]; then - rm -f /var/lib/apt/lists/lock /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/cache/apt/archives/lock 2>/dev/null - dpkg --configure -a 2>/dev/null || true - echo "CLEANUP: stale lock files removed, dpkg reconfigured" - else - echo "NOOP: no processes eligible for cleanup (use --extra-vars \"apt_force_cleanup=true\" to force)" - fi - register: apt_lock_cleanup - changed_when: "'KILLED' in apt_lock_cleanup.stdout | default('')" - failed_when: false - when: apt_lock_holders.stdout | default('') != "" - - - name: "Pre-flight: Re-check lock after cleanup" - ansible.builtin.shell: | - fuser /var/lib/apt/lists/lock /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/cache/apt/archives/lock 2>/dev/null - register: apt_lock_recheck - changed_when: false - failed_when: false - when: - - apt_lock_holders.stdout | default('') != "" - - "'KILLED' in apt_lock_cleanup.stdout | default('')" - - - name: "Pre-flight: Fail if APT lock persists after cleanup" - ansible.builtin.fail: - msg: | - ⛔ APT LOCK CONTENTION on {{ inventory_hostname }}: - Lock holder(s): {{ apt_lock_holders.stdout | default('(unable to detect)') }} - Running apt/dpkg: {{ apt_processes.stdout | default('(none)') }} - Process details: {{ apt_lock_process_info.stdout | default('(unable to inspect)') }} - Cleanup result: {{ apt_lock_cleanup.stdout | default('(no cleanup attempted)') }} - → APT systemd timers were auto-stopped by this playbook. - → Auto-cleanup was attempted for stale processes (>15 min runtime). - → To force-kill ALL lock holders, re-run with: - --extra-vars "apt_force_cleanup=true" - → Or SSH into {{ inventory_hostname }} and investigate manually. - when: - - apt_lock_holders.stdout | default('') != "" - - (apt_lock_cleanup.stdout | default('NOOP')) is not search('KILLED') or (apt_lock_recheck.stdout | default('')) != "" - -# ============================================================================== -# PHASE 1: BASE SYSTEM CONFIGURATION -# ============================================================================== -- name: "PHASE 1: Base System Configuration" - hosts: local - become: true - gather_facts: true - ignore_unreachable: true - any_errors_fatal: false - tags: [base, system, packages] - pre_tasks: - - name: "Validate supported OS distribution" - ansible.builtin.assert: - that: - - ansible_facts['distribution'] in ['Ubuntu', 'Debian', 'Pop', 'Pop!_OS'] - fail_msg: > - Unsupported OS distribution: {{ ansible_facts['distribution'] }}. - This playbook supports Ubuntu, Debian, and Pop OS (Ubuntu-based). - success_msg: "✅ Supported OS: {{ ansible_facts['distribution'] }} {{ ansible_facts['distribution_version'] }}" - - - name: "Normalize Pop OS to Ubuntu for apt/docker repos (Ubuntu-based)" - ansible.builtin.set_fact: - _normalized_distro: "{{ 'Ubuntu' if ansible_facts['distribution'] in ['Pop', 'Pop!_OS'] else ansible_facts['distribution'] }}" - _is_pop_os: "{{ true if ansible_facts['distribution'] in ['Pop', 'Pop!_OS'] else false }}" - - - name: "Pre-flight: Stop APT systemd timers" - ansible.builtin.systemd: - name: "{{ item }}" - state: stopped - enabled: false - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - failed_when: false - - - name: "Pre-flight: Track timer state for restore" - ansible.builtin.set_fact: - apt_timers_stopped: true - cacheable: false - - tasks: - - block: - - name: "Apply L1 OS baseline role" - ansible.builtin.include_role: - name: L1_os_baseline - rescue: - - name: "Recovery: Restore APT systemd timers" - ansible.builtin.systemd: - name: "{{ item }}" - state: started - enabled: true - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - when: apt_timers_stopped | default(false) - failed_when: false - - name: "Recovery: Clear timer tracking fact" - ansible.builtin.set_fact: - apt_timers_stopped: false - when: apt_timers_stopped | default(false) - -# ============================================================================== -# PHASE 2: SECURITY HARDENING (Workstation Profile) -# ============================================================================== -- name: "PHASE 2: Security & SSH Hardening" - hosts: local - become: true - gather_facts: false - ignore_unreachable: true - any_errors_fatal: false - tags: [security, firewall, ssh, fail2ban] - pre_tasks: - - name: "Pre-flight: Stop APT systemd timers" - ansible.builtin.systemd: - name: "{{ item }}" - state: stopped - enabled: false - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - failed_when: false - - - name: "Pre-flight: Track timer state for restore" - ansible.builtin.set_fact: - apt_timers_stopped: true - cacheable: false - - tasks: - - block: - - name: "Apply L2 compliance role (workstation profile)" - ansible.builtin.include_role: - name: L2_compliance - rescue: - - name: "Recovery: Restore APT systemd timers" - ansible.builtin.systemd: - name: "{{ item }}" - state: started - enabled: true - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - when: apt_timers_stopped | default(false) - failed_when: false - - name: "Recovery: Clear timer tracking fact" - ansible.builtin.set_fact: - apt_timers_stopped: false - when: apt_timers_stopped | default(false) - -# ============================================================================== -# PHASE 3: TAILSCALE VPN MESH (Manual Auth + Tag Validation) -# ============================================================================== -- name: "PHASE 3: Tailscale VPN Mesh" - hosts: local - become: true - gather_facts: true - ignore_unreachable: true - any_errors_fatal: false - tags: [vpn, tailscale, networking] - pre_tasks: - - name: "Pre-flight: Stop APT systemd timers" - ansible.builtin.systemd: - name: "{{ item }}" - state: stopped - enabled: false - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - failed_when: false - - - name: "Pre-flight: Track timer state for restore" - ansible.builtin.set_fact: - apt_timers_stopped: true - cacheable: false - - tasks: - - block: - - name: "Apply L2 tailscale client (tag:local, manual auth)" - ansible.builtin.include_role: - name: L2_compliance - rescue: - - name: "Recovery: Restore APT systemd timers" - ansible.builtin.systemd: - name: "{{ item }}" - state: started - enabled: true - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - when: apt_timers_stopped | default(false) - failed_when: false - - name: "Recovery: Clear timer tracking fact" - ansible.builtin.set_fact: - apt_timers_stopped: false - when: apt_timers_stopped | default(false) - -# ============================================================================== -# PHASE 4: DOCKER ENGINE -# ============================================================================== -- name: "PHASE 4: Docker Engine Installation" - hosts: local - become: true - gather_facts: false - ignore_unreachable: true - any_errors_fatal: false - tags: [docker, containers, runtime] - pre_tasks: - - name: "Pre-flight: Stop APT systemd timers" - ansible.builtin.systemd: - name: "{{ item }}" - state: stopped - enabled: false - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - failed_when: false - - - name: "Pre-flight: Track timer state for restore" - ansible.builtin.set_fact: - apt_timers_stopped: true - cacheable: false - - tasks: - - block: - - name: "Apply L6 runtime role (Docker Engine)" - ansible.builtin.include_role: - name: L6_runtime - rescue: - - name: "Recovery: Restore APT systemd timers" - ansible.builtin.systemd: - name: "{{ item }}" - state: started - enabled: true - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - when: apt_timers_stopped | default(false) - failed_when: false - - name: "Recovery: Clear timer tracking fact" - ansible.builtin.set_fact: - apt_timers_stopped: false - when: apt_timers_stopped | default(false) - - post_tasks: - - name: "Cleanup: Restore APT systemd timers" - ansible.builtin.systemd: - name: "{{ item }}" - state: started - enabled: true - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - when: apt_timers_stopped | default(false) - failed_when: false - - - name: "Cleanup: Clear timer tracking fact" - ansible.builtin.set_fact: - apt_timers_stopped: false - when: apt_timers_stopped | default(false) - -# ============================================================================== -# PHASE 5: DEVICE STATUS VERIFICATION -# ============================================================================== -- name: "PHASE 5: Device Status Verification" - hosts: local - gather_facts: false - become: true - ignore_unreachable: true - any_errors_fatal: false - tags: [verification, status] - tasks: - - name: Get Tailscale IP - ansible.builtin.command: tailscale ip -4 - register: ts_ip - changed_when: false - failed_when: false - - - name: Get Docker version - ansible.builtin.command: docker --version - register: docker_ver - changed_when: false - failed_when: false - - - name: Get UFW status - ansible.builtin.command: ufw status - register: ufw_status - changed_when: false - failed_when: false - - - name: Display device status - ansible.builtin.debug: - msg: - - "=== {{ inventory_hostname }} ===" - - "🔒 VPN IP: {{ ts_ip.stdout | default('N/A') }}" - - "🐳 {{ docker_ver.stdout | default('Docker not installed') }}" - - "🛡️ UFW: {{ ufw_status.stdout_lines[0] if (ufw_status.stdout_lines | default([])) | length > 0 else 'N/A' }}" - - "💻 Workstation ({{ server_type | default('workstation') }})" - - "🔑 Tailscale tag: tag:local" - -- name: "Local Device Hardening Complete" - hosts: localhost - gather_facts: true - become: false - tags: [always] - tasks: - - name: "Count reachable vs skipped devices" - ansible.builtin.set_fact: - _reachable_count: "{{ ansible_play_hosts_all | length }}" - _unreachable_hint: "Check Phase 0.15 diagnostics above for individual device errors" - - - name: Display hardening summary - ansible.builtin.debug: - msg: - - "" - - "==========================================" - - "LOCAL DEVICE HARDENING COMPLETE" - - "==========================================" - - "" - - " Devices in [local]: {{ groups['local'] | default([]) | length }}" - - "" - - " NOTE: Unreachable devices show warnings above." - - " Check Phase 0.15 for per-device SSH diagnostics." - - " Re-run this playbook when offline devices come online." - - "" - - " Security Profile (Workstation):" - - " ✅ SSH password auth DISABLED (keys only)" - - " ✅ Fail2ban active ({{ fail2ban_maxretry | default(5) }} attempts = {{ fail2ban_bantime | default('10m') }} ban)" - - " ✅ UFW enabled with {{ ufw_policy | default('allow') }} policy" - - " ✅ Tailscale VPN mesh - tag:local (manual auth)" - - " ✅ Tailscale ACL pre-applied (admin → tag:local:22, inter-device)" - - " ✅ Docker engine with daemon hardening" - - " ⏭️ CrowdSec skipped (not applicable to workstations)" - - "" - - " 🖥️ Supported distributions: Ubuntu, Debian, Pop OS" - - "" - - "==========================================" diff --git a/roles/L2_compliance/general/defaults/tailscale.yml b/roles/L2_compliance/general/defaults/tailscale.yml index 73126c6..1e681e6 100644 --- a/roles/L2_compliance/general/defaults/tailscale.yml +++ b/roles/L2_compliance/general/defaults/tailscale.yml @@ -6,7 +6,7 @@ # are configured in inventory/group_vars/all/tailscale.yml. Sensitive identities # and credentials reside in inventory/group_vars/all/secrets.sops.yml. -# Manual authentication bypass (e.g., local workstation devices) +# Manual authentication bypass (false by default; nodes require tailscale_auth_key) tailscale_manual_auth: false # Fallback node attributes (empty by default) diff --git a/roles/L2_compliance/general/tasks/security/nftables_rules.yml b/roles/L2_compliance/general/tasks/security/nftables_rules.yml index 55754f7..b2532a4 100644 --- a/roles/L2_compliance/general/tasks/security/nftables_rules.yml +++ b/roles/L2_compliance/general/tasks/security/nftables_rules.yml @@ -131,12 +131,6 @@ # Deny Caddy admin + Prometheus API on Tailscale (defense-in-depth) iifname tailscale0 tcp dport { 2019, 2020 } reject - - {% if hardening_profile == 'workstation' %} - # Workstation exceptions - udp dport 68 accept # DHCP client - udp dport 5353 accept # mDNS - {% endif %} } chain forward { diff --git a/roles/L2_compliance/general/tasks/tailscale/main.yml b/roles/L2_compliance/general/tasks/tailscale/main.yml index aaee37e..a1b7b8a 100644 --- a/roles/L2_compliance/general/tasks/tailscale/main.yml +++ b/roles/L2_compliance/general/tasks/tailscale/main.yml @@ -76,8 +76,7 @@ - name: Determine distribution ID for Tailscale ansible.builtin.set_fact: - _tailscale_distro: "{{ 'ubuntu' if ansible_facts['distribution'] | lower in ['pop', 'pop!_os'] else ansible_facts['distribution'] | lower }}" - distro_id: "{{ 'ubuntu' if ansible_facts['distribution'] | lower in ['pop', 'pop!_os'] else ansible_facts['distribution'] | lower }}" + distro_id: "{{ ansible_facts['distribution'] | lower }}" distro_release: "{{ ansible_facts['distribution_release'] }}" tags: [l2-compliance, tailscale] @@ -134,14 +133,14 @@ {{ '(auto-update enabled)' if (tailscale_version | default('*')) == '*' else '(pinned with dpkg hold)' }} tags: [l2-compliance, tailscale] -- name: Install WireGuard kernel modules (Ubuntu / Pop OS) +- name: Install WireGuard kernel modules (Ubuntu) ansible.builtin.apt: name: "linux-modules-extra-{{ ansible_facts['kernel'] }}" state: present lock_timeout: 600 allow_change_held_packages: true register: wireguard_pkg_install - when: ansible_facts['distribution'] in ['Ubuntu', 'Pop', 'Pop!_OS'] + when: ansible_facts['distribution'] == 'Ubuntu' failed_when: >- wireguard_pkg_install is failed and ('No package matching' not in (wireguard_pkg_install.msg | default(''))) and @@ -216,7 +215,7 @@ - name: Determine Tailscale ACL tag for this node ansible.builtin.set_fact: tailscale_acl_tag: >- - {{ 'tag:brain' if 'brain' in group_names else 'tag:muscle' if 'muscle' in group_names else 'tag:local' if 'local' in group_names else '' }} + {{ 'tag:brain' if 'brain' in group_names else 'tag:muscle' if 'muscle' in group_names else '' }} tags: [l2-compliance, tailscale] - name: Validate node has supported ACL tag mapping @@ -225,7 +224,7 @@ - tailscale_acl_tag | length > 0 fail_msg: >- Unable to derive Tailscale ACL tag for host {{ inventory_hostname }}. - Expected host membership in inventory group 'brain', 'muscle', or 'local'. + Expected host membership in inventory group 'brain' or 'muscle'. tags: [l2-compliance, tailscale] - name: Clear stale Tailscale state when status indicates NeedsLogin @@ -490,7 +489,6 @@ # (connection: local + become: false as play keywords). Task-level # become: false on delegated tasks is fragile because an ansible_become var # wins over task keywords (see ansible.cfg [privilege_escalation]). -# local-devices.yml applies the same chain from its own PHASE 0.1 play. # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ diff --git a/roles/L2_compliance/general/templates/tailscale/tailscale-acls.json.j2 b/roles/L2_compliance/general/templates/tailscale/tailscale-acls.json.j2 index 761fc3b..ad6610b 100644 --- a/roles/L2_compliance/general/templates/tailscale/tailscale-acls.json.j2 +++ b/roles/L2_compliance/general/templates/tailscale/tailscale-acls.json.j2 @@ -6,8 +6,6 @@ {% set acl_automation_sources = ((tailscale_acl_automation_sources | default(acl_operator_sources)) + (tailscale_acl_automation_sources_extra | default([]))) | unique %} {% set acl_breakglass_sources = (tailscale_acl_breakglass_sources | default([])) | unique %} {% set acl_admin_ui_sources = ((tailscale_acl_admin_ui_sources | default(acl_operator_sources)) + (tailscale_acl_admin_ui_sources_extra | default([]))) | unique %} -{% set acl_local_sources = ((tailscale_acl_local_sources | default(['autogroup:admin'])) + (tailscale_acl_local_sources_extra | default([]))) | unique %} -{% set has_local_devices = (groups.get('local', []) | length > 0) and (acl_local_sources | length > 0) %} {% set acl_rules = [] %} {% if acl_automation_sources | length > 0 %} {% set _ = acl_rules.append({ @@ -60,26 +58,6 @@ 'dst': ['tag:brain:8000', 'tag:brain:9443', 'tag:brain:9100', 'tag:brain:' ~ (observability_cadvisor_port | string), 'tag:brain:' ~ (tailscale_metrics_port | string)] }) %} -{% if has_local_devices %} -{% set local_host_dsts = [] %} -{% for host in groups['local'] %} -{% if hostvars[host].get('ansible_host', '') | length > 0 %} -{% set _ = local_host_dsts.append(host + ':22') %} -{% endif %} -{% endfor %} -{% set _ = acl_rules.append({ - 'action': 'accept', - 'src': acl_local_sources, - 'dst': ['tag:local:22'] + local_host_dsts - }) -%} -{% set _ = acl_rules.append({ - 'action': 'accept', - 'src': ['tag:local'], - 'dst': ['tag:local:22'] - }) -%} -{% endif %} {# self-allow: each admin user can reach own devices on any port #} {% if acl_automation_sources | length > 0 %} {% set _ = acl_rules.append({ @@ -94,8 +72,7 @@ "groups": {{ tailscale_acl_groups | to_json }}{% endif %}, "tagOwners": { "tag:brain": ["autogroup:admin"], - "tag:muscle": ["autogroup:admin"]{% if has_local_devices %}, - "tag:local": ["autogroup:admin"]{% endif %} + "tag:muscle": ["autogroup:admin"] }, "ssh": [], {# nodeAttrs managed via tailscale_node_attrs variable (default: none) #} diff --git a/roles/L6_runtime/general/tasks/main.yml b/roles/L6_runtime/general/tasks/main.yml index a1389a0..c438d11 100644 --- a/roles/L6_runtime/general/tasks/main.yml +++ b/roles/L6_runtime/general/tasks/main.yml @@ -9,22 +9,22 @@ # ============================================================================= # Installs Docker Engine with pinned versions for Docker Cluster # Arch auto-detected via ansible_facts['architecture'] fact -# Supports: Debian 11/12, Ubuntu 22.04/24.04, Pop OS (Ubuntu-based) +# Supports: Debian 11/12, Ubuntu 22.04/24.04 # Versions controlled centrally in group_vars/all/secrets.sops.yml # ============================================================================= -- name: Normalize Pop OS distribution to Ubuntu (Pop OS is Ubuntu-based) +- name: Set Docker distribution and release facts ansible.builtin.set_fact: - _docker_distro: "{{ 'Ubuntu' if ansible_facts['distribution'] in ['Pop', 'Pop!_OS'] else ansible_facts['distribution'] }}" + _docker_distro: "{{ ansible_facts['distribution'] }}" _docker_release: "{{ ansible_facts['distribution_release'] }}" - when: ansible_facts['distribution'] in ['Ubuntu', 'Debian', 'Pop', 'Pop!_OS'] + when: ansible_facts['distribution'] in ['Ubuntu', 'Debian'] tags: [l6-runtime, docker, compose] - name: Fail if OS is not a supported Docker distribution ansible.builtin.fail: msg: > Unsupported OS for Docker: {{ ansible_facts['distribution'] }}. - Supported: Ubuntu, Debian, Pop OS (all with apt-based Docker repos). + Supported: Ubuntu, Debian. when: _docker_distro is not defined tags: [l6-runtime, docker, compose] @@ -37,7 +37,6 @@ fail_msg: | docker_version_map must be defined in group_vars/all/secrets.sops.yml Current OS: {{ ansible_facts['distribution'] }} {{ _docker_release }} - (normalized to {{ _docker_distro }} for Pop OS compatibility) success_msg: "Docker version: {{ docker_version_map[_docker_distro][_docker_release] }}" tags: [l6-runtime, docker, compose] diff --git a/scripts/setup.sh b/scripts/setup.sh index 5ebf5ca..e5a3a2d 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -185,7 +185,6 @@ validate_playbooks() { playbooks/l6/engine.yml \ playbooks/l6/portainer.yml \ playbooks/ops/bootstrap.yml \ - playbooks/ops/local-devices.yml \ playbooks/ops/nuke.yml \ playbooks/ops/validate.yml; do if [[ -f "$pb" ]]; then From 2352f3f47ccb8f8713d96be163b2cff63f4420f8 Mon Sep 17 00:00:00 2001 From: Miguel Lozano Date: Wed, 4 Nov 2026 14:31:11 -0500 Subject: [PATCH 2/2] docs(security): establish zero-trust developer workstation baseline and align architecture --- docs/architecture/ARCHITECTURE.md | 59 ++++--- docs/architecture/LAYER_BOUNDARIES.md | 37 ++--- docs/operations/HOST_CLASSES.md | 154 ++++++------------- docs/operations/OPERATIONS_RUNBOOK.md | 5 +- docs/project/REPOSITORY_STRUCTURE.md | 4 +- docs/security/DEVELOPER_WORKSTATION_GUIDE.md | 98 ++++++++++++ docs/security/HARDENING-STATUS.md | 39 ++--- 7 files changed, 204 insertions(+), 192 deletions(-) create mode 100644 docs/security/DEVELOPER_WORKSTATION_GUIDE.md diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md index 8d7fc65..998d2c9 100644 --- a/docs/architecture/ARCHITECTURE.md +++ b/docs/architecture/ARCHITECTURE.md @@ -43,8 +43,6 @@ stack/ │ │ │ └── main.yml │ │ ├── muscle/ ← Server-grade, backups + observability │ │ │ └── main.yml -│ │ ├── local/ ← Workstation-grade, no backups, no observability -│ │ │ └── main.yml │ │ ├── app_/ ← Per-application profile overrides │ │ │ └── vars.yml ├── roles/ @@ -174,7 +172,6 @@ Layers MUST NOT reach across trust boundaries: | Package Manager | ✅ apt | ✅ apt | ✅ apt | ✅ apt | ✅ apt | ✅ apt | ✅ apt | ✅ apt | | Update/Hold Policies | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | | Auto-upgrade Checker | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | -| Pop OS → Ubuntu Norm. | ✅ Full | ✅ Full | - (N/A) | - (N/A) | - (N/A) | - (N/A) | - (N/A) | - (N/A) | | NIST 800-53 Hardening | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ✅ Full | | CIS Benchmarks | ✅ Full | ✅ Full | ✅ Full | ✅ Full | ⚠️ Minor delta | ⚠️ Minor delta | ⚠️ Minor delta | ⚠️ Minor delta | | Firewall | ✅ ufw | ✅ ufw | ✅ ufw | ✅ ufw | ⚠️ nftables wrapper | ⚠️ nftables wrapper | ⚠️ nftables wrapper | ⚠️ nftables wrapper | @@ -207,7 +204,7 @@ Layers MUST NOT reach across trust boundaries: ``` Layer 1: roles//defaults/main.yml ← Lowest priority - role fallbacks Layer 2: group_vars/all/main.yml ← Global safe defaults -Layer 3: group_vars/{brain,muscle,local}/main.yml ← Host-class config +Layer 3: group_vars/{brain,muscle}/main.yml ← Host-class config Layer 4: group_vars/app_/vars.yml ← Application-level overrides ``` @@ -235,13 +232,12 @@ Layer 4: group_vars/app_/vars.yml ← Application-level override ### Dispatcher Pattern ``` -hardening_profile: server | workstation +hardening_profile: server ``` -- **`server`**: Full hardening - server-grade firewall (`ufw deny`/`nftables deny`), fail2ban 1h bantime, CrowdSec enforcement, observability stack, backups enabled. -- **`workstation`**: Tailored hardening - UFW deny (Zero Trust base), fail2ban 10m bantime, CrowdSec skipped (false positives from network changes), no observability, no backups. +- **`server`**: Full server hardening - in-kernel nftables firewall (`policy drop`), fail2ban brute-force protection, CrowdSec collaborative threat intelligence, VictoriaMetrics/Grafana observability stack, Restic backup automation. -The dispatcher is a single variable in `group_vars/{brain,muscle,local}/main.yml`. The `local` host class sets `hardening_profile: workstation`; `brain` and `muscle` set `server`. +The dispatcher is configured per host class in `group_vars/{brain,muscle}/main.yml` and defaults globally to `server` in `group_vars/all/main.yml`. Developer and operator workstation security follows a Zero Trust model outside Ansible management (see [DEVELOPER_WORKSTATION_GUIDE.md](../security/DEVELOPER_WORKSTATION_GUIDE.md)). --- @@ -344,21 +340,21 @@ Target Host: /srv/evidence/nist// --- -## 7. Host Classes - brain / muscle / local +## 7. Host Classes - brain / muscle ### Capability Matrix -| Capability | `brain` | `muscle` | `local` | -| --------------------- | ----------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -------------------------------------------------------------- | -| **Backups** | ✅ Enabled - app data (L5) + runtime state (L6) | ✅ Enabled - app data (L5) + runtime state (L6) | ❌ Disabled | -| **Observability** | ✅ Full stack (VictoriaMetrics, Grafana, Loki) | ✅ Full stack (VictoriaMetrics, Grafana, Loki) | ❌ Disabled | -| **Hardening Profile** | Server-grade (`hardening_profile: server`) | Server-grade (`hardening_profile: server`) | Workstation-grade (`hardening_profile: workstation`) | -| **Resource Profile** | Management node - moderate CPU, persistent storage for backups + evidence | Compute worker - higher CPU/RAM for application workloads, persistent storage for data | Personal workstation - variable, no guaranteed resources | -| **Failure Domain** | Single point of failure for management (Portainer, Grafana, monitoring). Redundancy via multiple brains possible. | Stateless apps - any muscle node can be replaced. | Independent - failure does not affect other nodes | -| **CrowdSec** | ✅ Installed (collaborative IPS) | ✅ Installed (collaborative IPS) | ❌ Skipped (false positives from network changes) | -| **Tailscale Auth** | Automated (`--authkey`) | Automated (`--authkey`) | Manual (`tailscale up` by user) | -| **UFW Policy** | Default deny (`policy: deny`) | Default deny (`policy: deny`) | Deny (Zero Trust base layer; inbound denied, outbound allowed) | -| **Fail2ban Bantime** | 1 hour | 1 hour | 10 minutes (avoids lockout without IPMI/console) | +| Capability | `brain` | `muscle` | +| --------------------- | ----------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| **Backups** | ✅ Enabled - app data (L5) + runtime state (L6) | ✅ Enabled - app data (L5) + runtime state (L6) | +| **Observability** | ✅ Full stack (VictoriaMetrics, Grafana, Loki) | ✅ Full stack (VictoriaMetrics, Grafana, Loki) | +| **Hardening Profile** | Server-grade (`hardening_profile: server`) | Server-grade (`hardening_profile: server`) | +| **Resource Profile** | Management node - moderate CPU, persistent storage for backups + evidence | Compute worker - higher CPU/RAM for application workloads, persistent storage for data | +| **Failure Domain** | Single point of failure for management (Portainer, Grafana, monitoring). Redundancy via multiple brains possible. | Stateless apps - any muscle node can be replaced. | +| **CrowdSec** | ✅ Installed (collaborative IPS) | ✅ Installed (collaborative IPS) | +| **Tailscale Auth** | Automated (`--authkey`) | Automated (`--authkey`) | +| **Firewall Policy** | Default deny (`policy drop`, nftables) | Default deny (`policy drop`, nftables) | +| **Fail2ban Bantime** | 1 hour | 1 hour | ### Allowed Roles per Class @@ -366,15 +362,15 @@ Target Host: /srv/evidence/nist// | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | | **brain** | `L1_os_baseline/general`, `L2_compliance/general/tasks/security`, `L2_compliance/general` (user_hardening, tailscale), `L2_compliance/nist_800_53` (CrowdSec, compliance), `L3_observability/general`, `L4_networking/caddy`, `L6_runtime/docker_compose`, `L6_runtime/portainer` (optional), `L6_runtime/backup` | 1 per platform (or ≥2 for HA) | | **muscle** | `L1_os_baseline/general`, `L2_compliance/general/tasks/security`, `L2_compliance/general` (user_hardening, tailscale), `L2_compliance/nist_800_53` (CrowdSec, compliance), `L3_observability/general`, `L4_networking/caddy`, `L6_runtime/docker_compose`, `L6_runtime/portainer` (optional, Edge Agent), `L6_runtime/backup` | N (scale horizontally) | -| **local** | `L1_os_baseline/general`, `L2_compliance/general/tasks/security` (workstation profile), `L2_compliance/general` (tailscale, manual auth, `tag:local`), `L6_runtime/docker_compose` | Per operator | + +> **Note on Workstation Devices**: Operator and developer endpoints are external client nodes connecting via Tailscale; they are not provisioned or managed through Ansible host classes. See [DEVELOPER_WORKSTATION_GUIDE.md](../security/DEVELOPER_WORKSTATION_GUIDE.md). ### Host Class Selection -Determined by `group_vars/{brain,muscle,local}/main.yml`. A host's inventory group membership determines which group_vars are applied. The `hardening_profile` variable is set per class: +Determined by `group_vars/{brain,muscle}/main.yml`. A host's inventory group membership determines which group_vars are applied: -- `brain/main.yml` → `hardening_profile: server` -- `muscle/main.yml` → `hardening_profile: server` -- `local/main.yml` → `hardening_profile: workstation` +- `brain/main.yml` → Management node tunables +- `muscle/main.yml` → Worker node tunables ### Operational Scale & Sizing Boundaries @@ -382,12 +378,12 @@ This repository is engineered under the **"Boring is Better"** philosophy: provi #### Hardware Profiles & Node Allocation -| Host Class | Hardware Profile | Recommended Count | Architectural Role & Workload Distribution | -| :--- | :--- | :--- | :--- | -| **`brain`** | **High-Spec** (8–16 vCPU, 32–64 GB RAM) | **1 – 3 nodes** | Centralized control plane: VictoriaMetrics, Loki, Grafana, Portainer. 3 nodes provide active quorum for HA. | -| **`brain`** | **Low-Spec** (2–4 vCPU, 4–8 GB RAM) | **3 – 5 nodes** | Role-partitioned control plane: dedicated nodes for Ingress/VPN, metrics ingestion, log indexing (Loki), and management across low-cost VPS instances. | -| **`muscle`** | **Any compute tier** | **1 – 50 nodes** | Standard push-based Ansible sweet spot. Full parallel execution with low drift and minimal latency. | -| **`muscle`** | **Any compute tier** | **50 – 100 nodes** | Upper operational ceiling for push-based Ansible. Deployments should target specific nodes or tags (`--limit muscle-X`). | +| Host Class | Hardware Profile | Recommended Count | Architectural Role & Workload Distribution | +| :----------- | :-------------------------------------- | :----------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`brain`** | **High-Spec** (8–16 vCPU, 32–64 GB RAM) | **1 – 3 nodes** | Centralized control plane: VictoriaMetrics, Loki, Grafana, Portainer. 3 nodes provide active quorum for HA. | +| **`brain`** | **Low-Spec** (2–4 vCPU, 4–8 GB RAM) | **3 – 5 nodes** | Role-partitioned control plane: dedicated nodes for Ingress/VPN, metrics ingestion, log indexing (Loki), and management across low-cost VPS instances. | +| **`muscle`** | **Any compute tier** | **1 – 50 nodes** | Standard push-based Ansible sweet spot. Full parallel execution with low drift and minimal latency. | +| **`muscle`** | **Any compute tier** | **50 – 100 nodes** | Upper operational ceiling for push-based Ansible. Deployments should target specific nodes or tags (`--limit muscle-X`). | #### Scaling Beyond 100 Nodes (Ecosystem Positioning) @@ -398,6 +394,7 @@ This repository is engineered under the **"Boring is Better"** philosophy: provi #### Ingress Routing Catalog & Secrets Architecture Routing topology and secrets adhere to strict separation of concerns: + - **Routing Catalog** (`inventory/group_vars/all/ingress.yml`): Plain YAML catalog of domains, upstreams, headers, and security profiles. Ignored by `.gitignore` to prevent leaking private client infrastructure in open-source environments (template provided in `ingress.yml.example`). - **Cryptographic Secrets** (`inventory/group_vars/all/secrets.sops.yml`): Strictly reserved for sensitive credentials (TLS private keys in `cloudflare_origin_certs`, `caddy_ui_token`, R2 access keys). - **Ingress Template** (`roles/L4_networking/caddy/templates/Caddyfile.j2`): 100% data-driven, version-controlled Jinja2 template with zero hardcoded values, featuring automatic global `metrics` export and per-service WAF/TLS client routing. diff --git a/docs/architecture/LAYER_BOUNDARIES.md b/docs/architecture/LAYER_BOUNDARIES.md index 9701d9a..04bad43 100644 --- a/docs/architecture/LAYER_BOUNDARIES.md +++ b/docs/architecture/LAYER_BOUNDARIES.md @@ -109,7 +109,7 @@ OpenSource surface. Audit trail: `docs/compliance/evidence/L0_AUDIT.md`. L1 standardizes the operating system. It is the **bottom of the OpenSource surface** - the first layer that any Ansible playbook touches. L1 owns: package manager configuration (apt sources, pinning), update/hold policies, -auto-upgrade checker, OS family detection, Pop OS → Ubuntu normalization, +auto-upgrade checker, OS family detection and validation, timezone, locale, and kernel module blacklisting (`cramfs`, `freevxfs`, etc.). L1 MUST detect the host architecture (amd64/arm64) and make it available as @@ -119,7 +119,7 @@ L1 MUST detect the host architecture (amd64/arm64) and make it available as 1. **Package manager is configured and consistent** - apt sources point to reliable mirrors, hold pins are applied, no stale or broken packages. 2. **Update policies are active** - unattended-upgrades configured per host class. Security updates auto-applied; non-security updates follow declared policy. -3. **OS family is normalized** - Pop OS is detected and remapped to Ubuntu. `ansible_distribution` and `ansible_os_family` are reliable facts for upper layers. +3. **OS family is validated** - distribution is validated against supported targets (Debian 12, Ubuntu 22.04). `ansible_distribution` and `ansible_os_family` are reliable facts for upper layers. 4. **Kernel modules are clean** - insecure/unnecessary modules (`cramfs`, `freevxfs`, `hfs`, `hfsplus`, `squashfs`, `udf`) are blacklisted. 5. **Architecture is declared** - `ansible_architecture` is set (amd64 or arm64) and available as a fact for all upper layers. 6. **Time and locale are correct** - timezone is set, NTP is active (via systemd-timesyncd or chrony). @@ -152,13 +152,12 @@ well-understood layer: it applies the same logic regardless of host class. ### Contract Breach -| Failure Mode | Consequence | Detection | -| -------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -| Package manager broken (404, unsigned repos) | L2 cannot install `ufw`, `fail2ban`, CrowdSec. L6 cannot install Docker. | `apt update` → non-zero exit. `playbooks/site.yml` fails at L1. | -| Unattended-upgrades not configured | Security patches not applied. CVEs accumulate. | `compliance.yml` checks `unattended-upgrades` status. L3 alerts if packages held > 30 days. | -| Pop OS not normalized | `ansible_distribution` returns "Pop!\_OS" instead of "Ubuntu". OS-specific `when:` conditions fail in L2. | `gather_facts` output shows unexpected `ansible_distribution`. `playbooks/site.yml` fails at L2 with undefined variables. | -| Kernel modules not blacklisted | `cramfs`, `freevxfs` loadable. NIST control SC-7 violation. | `compliance.yml` checks `/etc/modprobe.d/blacklist.conf`. Evidence missing from `/srv/evidence/nist/sc-7/`. | -| Wrong timezone/NTP | Log timestamps are incorrect. TLS certificate validation may fail. Grafana dashboards show wrong times. | L3 monitoring detects clock skew. Caddy logs show `x509: certificate has expired or is not yet valid`. | +| Failure Mode | Consequence | Detection | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| Package manager broken (404, unsigned repos) | L2 cannot install `ufw`, `fail2ban`, CrowdSec. L6 cannot install Docker. | `apt update` → non-zero exit. `playbooks/site.yml` fails at L1. | +| Unattended-upgrades not configured | Security patches not applied. CVEs accumulate. | `compliance.yml` checks `unattended-upgrades` status. L3 alerts if packages held > 30 days. | +| Kernel modules not blacklisted | `cramfs`, `freevxfs` loadable. NIST control SC-7 violation. | `compliance.yml` checks `/etc/modprobe.d/blacklist.conf`. Evidence missing from `/srv/evidence/nist/sc-7/`. | +| Wrong timezone/NTP | Log timestamps are incorrect. TLS certificate validation may fail. Grafana dashboards show wrong times. | L3 monitoring detects clock skew. Caddy logs show `x509: certificate has expired or is not yet valid`. | --- @@ -179,20 +178,19 @@ enforcement vs detection-only toggle), OCI killswitch (conditional, Ubuntu OCI only), NIST/CIS evidence collection to `/srv/evidence/`, and Tailscale client enrollment. -The `hardening_profile` dispatcher (`server` | `workstation`) controls -intensity: `server` applies full hardening (CrowdSec enforcement, fail2ban 1h -bantime, strict UFW deny), `workstation` applies tailored hardening -(CrowdSec skipped, fail2ban 10m bantime, UFW deny with relaxed outbound). +The `hardening_profile` dispatcher (`server`) applies server-grade +hardening: CrowdSec enforcement, fail2ban brute-force protection, +in-kernel nftables default-drop firewall, and strict sysctl tuning. ### Promises 1. **SSH is hardened** - password auth disabled, root login disabled, key-only access, strict cipher/MAC/KEX algorithms. -2. **Firewall is active and enforcing** - default deny inbound. Only declared ports are open. Backend auto-selected: ufw on Ubuntu, nftables on Debian 12. +2. **Firewall is active and enforcing** - default deny inbound (`policy drop`). Only declared ports are open via unified in-kernel nftables. 3. **fail2ban is running** - configured per host class. Protects SSH from brute-force. -4. **CrowdSec is operational** - installed and configured. Enforcement or detection-only per `hardening_profile`. +4. **CrowdSec is operational** - installed and configured. 5. **Kernel is hardened** - sysctl parameters applied (`net.ipv4.tcp_syncookies`, `kernel.kptr_restrict`, etc.). NIST CM-7 controls verified. 6. **Compliance evidence is collected** - machine-generated evidence files written to `/srv/evidence/nist//`. Traceable to Ansible task tags. -7. **Tailscale is enrolled** - host is connected to the Tailscale mesh. `brain`/`muscle` use automated auth; `local` uses manual auth. +7. **Tailscale is enrolled** - host is connected to the Tailscale mesh with automated node auth. ### Forbids @@ -258,8 +256,7 @@ Loki (log aggregation), scrape target configuration, health check endpoints, and alerting rules. L3 is the **monitoring layer** - it watches everything above and below it without modifying it. -L3 is deployed only on `brain` and `muscle` host classes. `local` hosts skip -L3 entirely. +L3 is deployed across `brain` and `muscle` host classes. ### Promises @@ -431,7 +428,7 @@ use to deploy their own application instances. - MUST NOT configure monitoring targets. That is L3 (L5 only declares `monitoring.health_endpoint` - L3 acts on it). - MUST NOT configure Caddy ingress routes. That is L4 (L5 only provides ingress service data - L4 renders it). - MUST NOT store application secrets in committed files. `.env.example` carries `changeme` placeholders; the operator provides real values in `.env`. -- MUST NOT import variables from `group_vars/{brain,muscle,local}/` - host-class variables belong to the host, not the app. +- MUST NOT import variables from `group_vars/{brain,muscle}/` - host-class variables belong to the host, not the app. - MUST NOT reference files that do not exist (`compose.yml.j2`, `vars.yml`, `secrets.yml` are not part of the standardized layout). ### Depends On @@ -463,7 +460,7 @@ that operators rely on when running applications. | `supported_arch` mismatch | Profile declares arch the target host does not support. | Pre-flight arch check. Error: "Host arch 'arm64' not in supported_arch ['amd64']". | | `changeme` placeholder reaches runtime | Containers start with default/example credentials. | Operator review on deploy; `detect-secrets` scan of `apps/`. | | Backup job fails silently | App data not backed up. Data loss on disk failure. | L3 alert: backup job exit code ≠ 0. Grafana dashboard shows "last backup age" > schedule interval. | -| App profile references host-class vars | Profile picks up wrong port/firewall config from host group. Cross-layer coupling. | Lint rule: profiles must not import `group_vars/{brain,muscle,local}/` variables. | +| App profile references host-class vars | Profile picks up wrong port/firewall config from host group. Cross-layer coupling. | Lint rule: profiles must not import `group_vars/{brain,muscle}/` variables. | --- diff --git a/docs/operations/HOST_CLASSES.md b/docs/operations/HOST_CLASSES.md index 5875289..9f13101 100644 --- a/docs/operations/HOST_CLASSES.md +++ b/docs/operations/HOST_CLASSES.md @@ -4,7 +4,7 @@ type: operations owner: maintainers audience: operator version: v6.0.0 -last-reviewed: 2026-07-16 +last-reviewed: 2026-11-04 status: active project: developmi-hardened repo: github.com/Developmi/hardened @@ -12,7 +12,7 @@ repo: github.com/Developmi/hardened # How to Assign and Manage Host Classes -This guide walks you through assigning hosts to classes (`brain`, `muscle`, `local`), migrating hosts between classes, and verifying the result. Host classes control which Ansible roles, hardening profiles, and platform services apply to a node. Assigning the wrong class means the wrong profile is deployed. +This guide walks you through assigning hosts to classes (`brain`, `muscle`), migrating hosts between classes, and verifying the result. Host classes control which Ansible roles, hardening profiles, and platform services apply to a node. Assigning the wrong class means the wrong profile is deployed. _(Source: ARCHITECTURE.md §7)_ @@ -20,13 +20,13 @@ _(Source: ARCHITECTURE.md §7)_ You need to understand what each class does before assigning one. -**Capability summary** - `brain` nodes are management (Portainer, Grafana, backups); `muscle` nodes run application workloads; `local` nodes are operator workstations with minimal platform footprint. For the full 10-row capability matrix (backups, observability, CrowdSec, Tailscale auth, UFW policy, fail2ban bantime, etc.), see [ARCHITECTURE.md §7 - Capability Matrix](../architecture/ARCHITECTURE.md#7-host-classes-brain-muscle-local). +**Capability summary** - `brain` nodes are management (Portainer, Grafana, backups, ingress control); `muscle` nodes run application workloads. For the full capability matrix (backups, observability, CrowdSec, Tailscale auth, nftables policy, fail2ban bantime, etc.), see [ARCHITECTURE.md §7 - Capability Matrix](../architecture/ARCHITECTURE.md#7-host-classes-brain-muscle). -**Role summary** - `brain` and `muscle` get the full L1-L6 role stack (with brain-only management roles), `local` gets L1-L2 plus Docker engine only. For the complete per-class allowed-roles table, see [ARCHITECTURE.md §7 - Allowed Roles per Class](../architecture/ARCHITECTURE.md#7-host-classes-brain-muscle-local). +**Role summary** - `brain` and `muscle` get the full L1-L6 role stack (with brain-only management roles). For the complete per-class allowed-roles table, see [ARCHITECTURE.md §7 - Allowed Roles per Class](../architecture/ARCHITECTURE.md#7-host-classes-brain-muscle). -**Hardening profile** - the `hardening_profile` dispatcher (in [HARDENING-STATUS.md](../security/HARDENING-STATUS.md#hardening-profile-dispatcher)) routes `brain`/`muscle` → `server`, `local` → `workstation`. +**Hardening profile** - the `hardening_profile` dispatcher routes `brain` and `muscle` to `server`. -> **Legacy variables (documented, not used):** `node_role` (still present in `inventory/hosts.ini.example`), `server_role` and `server_type` (in `inventory/group_vars/{brain,muscle,local}/main.yml`) are legacy classifiers with no code usage. The real classifiers are `host_class` (inventory group membership) + `hardening_profile`. Do not build new logic on the legacy vars; `server_type` is only consumed as a cosmetic display string in `playbooks/ops/local-devices.yml`. They are kept in the repo for reference only and may be removed in a future major release. +> **Note on Workstation Devices**: Operator and developer laptops/workstations are client endpoints connecting to the cluster over Tailscale; they are not managed through Ansible host classes. See [DEVELOPER_WORKSTATION_GUIDE.md](../security/DEVELOPER_WORKSTATION_GUIDE.md). **Brain vs muscle roles** - both run Docker, but with different purposes: @@ -39,8 +39,8 @@ The suite does NOT deploy applications - it prepares the hardened foundation. Ap ### Prerequisites -- Ansible inventory file (`hosts.yml` or equivalent) is writable -- `inventory/group_vars/` directory structure exists: `brain/`, `muscle/`, `local/` +- Ansible inventory file (`inventory/hosts.ini` or equivalent) is writable +- `inventory/group_vars/` directory structure exists: `brain/`, `muscle/` - SOPS age private key available at `~/.config/sops/age/keys.txt` to decrypt `secrets.sops.yml` (the Tailscale trio in `secrets.yml` is the only vaulted exception until gate D4, OPERATIONS_RUNBOOK §7.4) - Bootstrap identity follows the inventory: SSH as `root` on classic VPS/lab hosts; on cloud images (OCI/AWS/GCP) SSH as the cloud image user (non-root, NOPASSWD sudo). `ansible_become` is derived from `ansible_user`. Declare `ansible_user` per host and `ansible_ssh_private_key_file` in `all:vars`. @@ -55,53 +55,40 @@ hardening_profile: server EOF ``` -Repeat for `muscle` or `local`, setting `hardening_profile: server` or `hardening_profile: workstation` respectively. +Repeat for `muscle`, setting `hardening_profile: server`. ### Step 2 - Tag the host in inventory Add the host to the matching Ansible group in your inventory file: -```yaml -# inventory/hosts.yml -all: - children: - brain: - hosts: - mgmt-node-01: - ansible_host: 10.0.0.10 - muscle: - hosts: - worker-01: - ansible_host: 10.0.0.20 - worker-02: - ansible_host: 10.0.0.21 - local: - hosts: - dev-laptop: - ansible_host: 192.168.1.50 +```ini +# inventory/hosts.ini +[brain] +brain-1 ansible_host=100.125.56.109 ansible_user=automation public_ip=178.104.18.57 node_role=management caddy_node_id=brain-1 + +[muscle] +muscle-1 ansible_host=100.66.224.76 ansible_user=automation public_ip=150.136.240.35 node_role=worker caddy_node_id=muscle-1 ``` ### Step 3 - Verify group_vars are loaded ```bash # Confirm the host picks up the right hardening_profile -ansible mgmt-node-01 -m debug -a "var=hardening_profile" +uv run ansible mgmt-node-01 -m debug -a "var=hardening_profile" # List all facts for the host to verify group membership -ansible mgmt-node-01 -m setup | grep ansible_playbook_dir +uv run ansible mgmt-node-01 -m setup | grep ansible_playbook_dir ``` ### Step 4 - Dry-run with the new class ```bash -uv run ansible-playbook -i inventory/hosts.yml site.yml --limit mgmt-node-01 --check --diff +uv run ansible-playbook -i inventory/hosts.ini playbooks/l2/hardening.yml --limit mgmt-node-01 --check --diff ``` -If the dry-run shows unexpected changes (e.g., a `local` host picking up server-grade hardening), recheck the inventory group membership and `group_vars` file path. - ## Class Migration Procedure -Migrating a host between classes (e.g., promoting a `local` node to `muscle`, or downgrading a `brain` → `muscle`) requires careful reconfiguration - the hardening profile and allowed roles change. +Migrating a host between classes (`brain` ↔ `muscle`) requires careful reconfiguration. ### brain → muscle migration @@ -115,25 +102,20 @@ Migrating a host between classes (e.g., promoting a `local` node to `muscle`, or --flat ``` -2. **Reassign group in inventory** - move the host from the `brain` group to the `muscle` group in `inventory/hosts.yml`. If the host is the _only_ brain, you must first designate another node as brain or accept that management services (Portainer, Grafana) will go down. +2. **Reassign group in inventory** - move the host from the `brain` group to the `muscle` group in `inventory/hosts.ini`. If the host is the _only_ brain, you must first designate another node as brain or accept that management services (Portainer, Grafana) will go down. -3. **Review role differences** - `muscle` excludes management-only roles. Verify `site.yml` or your playbook does not assign brain-only roles (e.g., `portainer` as primary server) to this host. +3. **Review role differences** - `muscle` excludes management-only roles. Verify your playbooks do not assign brain-only roles (e.g., `portainer` as primary server) to this host. 4. **Dry-run the migration** ```bash - uv run ansible-playbook -i inventory/hosts.yml site.yml \ + uv run ansible-playbook -i inventory/hosts.ini playbooks/l2/hardening.yml \ --limit --check --diff ``` - Watch for: - - Firewall rule deltas (UFW policy may not change between brain/muscle, but verify) - - Role removals that would stop services (Portainer server, observability dashboards) - - Backup target reassignment - 5. **Apply** ```bash - uv run ansible-playbook -i inventory/hosts.yml site.yml --limit + uv run ansible-playbook -i inventory/hosts.ini playbooks/l2/hardening.yml --limit ``` ### muscle → brain migration @@ -144,15 +126,6 @@ Same steps in reverse, plus: - Verify observability dashboards are reachable after migration - If HA, add the new brain to the HA configuration -### local → muscle migration - -**This is a significant class change** - the host goes from workstation-grade hardening to server-grade. Steps: - -1. Apply muscle `group_vars` per the assignment procedure above -2. Dry-run first - expect SSH config changes, UFW policy tightening, and new service installations (CrowdSec, observability agents, backup timers) -3. Ensure Tailscale auth switches from manual to automated (provision `--authkey`) -4. Apply with `--limit` to avoid unintended changes on other hosts - ## Verification Checklist Run these commands after assigning a class or after a migration to confirm the host is in the expected state. @@ -161,54 +134,35 @@ Run these commands after assigning a class or after a migration to confirm the h ```bash # 1. Confirm hardening_profile is correct -ansible -m debug -a "var=hardening_profile" -# Expected: "server" for brain/muscle, "workstation" for local +uv run ansible -m debug -a "var=hardening_profile" +# Expected: "server" for brain/muscle # 2. Check active roles (via ansible-playbook tag report) -uv run ansible-playbook -i inventory/hosts.yml site.yml \ +uv run ansible-playbook -i inventory/hosts.ini playbooks/l2/hardening.yml \ --limit --list-tags # 3. Verify group membership -ansible -m debug -a "var=group_names" -# Expected: ["brain"], ["muscle"], or ["local"] +uv run ansible -m debug -a "var=group_names" +# Expected: ["brain"] or ["muscle"] -# 4. Check UFW status (brain/muscle) -ansible -m shell -a "ufw status verbose" --become +# 4. Check nftables firewall status +uv run ansible -m shell -a "systemctl is-active nftables && nft list ruleset | grep 'table inet filter'" --become -# 5. Check fail2ban status (all classes) -ansible -m shell -a "fail2ban-client status" --become +# 5. Check fail2ban status +uv run ansible -m shell -a "fail2ban-client status" --become -# 6. Check CrowdSec (brain/muscle only) -ansible -m shell -a "cscli metrics" --become +# 6. Check CrowdSec +uv run ansible -m shell -a "cscli metrics" --become # 7. Check Tailscale connectivity -ansible -m shell -a "tailscale status" --become - -# 8. Full system facts snapshot (compare before/after migration) -ansible -m setup > facts_$(date +%Y%m%d).json -``` - -### Sample verification output - -```bash -$ ansible mgmt-node-01 -m debug -a "var=hardening_profile" -mgmt-node-01 | SUCCESS => { - "hardening_profile": "server" -} - -$ ansible mgmt-node-01 -m debug -a "var=group_names" -mgmt-node-01 | SUCCESS => { - "group_names": [ - "brain" - ] -} +uv run ansible -m shell -a "tailscale status" --become ``` ## Troubleshooting ### Wrong hardening profile applied -**Symptom:** `hardening_profile` returns `workstation` on a server host (or vice versa). +**Symptom:** `hardening_profile` returns unexpected value on a server host. **Causes:** @@ -219,38 +173,20 @@ mgmt-node-01 | SUCCESS => { ```bash # Check which vars files are loaded -ansible -m debug -a "var=vars" +uv run ansible -m debug -a "var=vars" # Verify the group file exists -ls -la inventory/group_vars/{brain,muscle,local}/main.yml +ls -la inventory/group_vars/{brain,muscle}/main.yml ``` ### Missing group_vars file **Symptom:** `hardening_profile` is undefined, playbook fails with `'hardening_profile' is undefined`. -**Fix:** Create the file per the assignment procedure above, or add a default in `inventory/group_vars/all/main.yml` (with a loud comment that it must be overridden): +**Fix:** Create the file per the assignment procedure above, or rely on global default in `inventory/group_vars/all/main.yml`: ```yaml # inventory/group_vars/all/main.yml -# ponytail: fallback default; every host SHOULD set this in its class group_vars -hardening_profile: "UNDEFINED - set per host class!" -``` - -### Role mismatch - -**Symptom:** A `local` host runs CrowdSec or backup roles it should not have. - -**Cause:** `site.yml` or playbook does not gate roles on group membership. - -**Fix:** Add `when: "'local' not in group_names"` conditionals in the playbook, or restructure plays so roles are applied per group: - -```yaml -- hosts: brain:muscle - roles: - - crowdsec - - observability - - backup - - backup-timers +hardening_profile: server ``` ### Role removed but service still running @@ -260,13 +196,13 @@ hardening_profile: "UNDEFINED - set per host class!" **Fix:** Ansible roles are additive by default - they do not automatically stop services when a role is removed from a host. Manually stop and disable: ```bash -ansible -m systemd -a "name=portainer state=stopped enabled=no" --become +uv run ansible -m systemd -a "name=portainer state=stopped enabled=no" --become ``` ## Cross-References -- [ARCHITECTURE.md §7 - Host Classes and Capability Profiles](../architecture/ARCHITECTURE.md#7-host-classes-brain-muscle-local) - SSOT for capability matrix and allowed roles +- [ARCHITECTURE.md §7 - Host Classes and Capability Profiles](../architecture/ARCHITECTURE.md#7-host-classes-brain-muscle) - SSOT for capability matrix and allowed roles - [GLOSSARY.md - Host Classes](../GLOSSARY.md#host-classes) - Platform terminology -- [HARDENING-STATUS.md - Hardening Profile Dispatcher](../security/HARDENING-STATUS.md#hardening-profile-dispatcher) - Profile dispatch logic -- [LAYER_BOUNDARIES.md](../architecture/LAYER_BOUNDARIES.md) - L3 deployment scoping +- [DEVELOPER_WORKSTATION_GUIDE.md](../security/DEVELOPER_WORKSTATION_GUIDE.md) - Workstation security baseline +- [LAYER_BOUNDARIES.md](../architecture/LAYER_BOUNDARIES.md) - Layer deployment scoping - [OPERATIONS_RUNBOOK.md](OPERATIONS_RUNBOOK.md) - Boot sequence per layer diff --git a/docs/operations/OPERATIONS_RUNBOOK.md b/docs/operations/OPERATIONS_RUNBOOK.md index 256ff9c..24c85b7 100644 --- a/docs/operations/OPERATIONS_RUNBOOK.md +++ b/docs/operations/OPERATIONS_RUNBOOK.md @@ -109,7 +109,7 @@ Makefile handles prompting automatically in a scalable way: - If inventory contains a host with `ansible_user` different from `root`, it automatically adds `--ask-become-pass`. - If all hosts are `root`, it does not add become password prompt. -- Deploy targets do NOT pass `--ask-vault-pass` by default: runtime secrets decrypt transparently via SOPS+age. Only the six Tailscale trio consumers pin the flag until the trio expires (D4): `deploy-hardening`, `deploy-compliance`, `reconnect-tailscale`, `deploy-local`, `nuke`, `provision-host`. +- Deploy targets do NOT pass `--ask-vault-pass` by default: runtime secrets decrypt transparently via SOPS+age. Only the five Tailscale trio consumers pin the flag until the trio expires (D4): `deploy-hardening`, `deploy-compliance`, `reconnect-tailscale`, `nuke`, `provision-host`. Examples: @@ -1013,7 +1013,6 @@ Run individual playbooks for targeted operations. | `l3/stack.yml` | `make deploy-monitoring` | VictoriaMetrics + Grafana + Loki (brain only) | | `l6/backup-appdata.yml` | `make deploy-backups` | App data backup (database dumps → Restic → R2) | | `l6/backup-engine.yml` | `make deploy-backup-engine` | Systemd backup timer and engine scheduling | -| `local-devices.yml` | `make deploy-local` | Workstation hardening | | `l2/compliance.yml` | `make deploy-compliance` | Compliance audit tags only | ### 5.3 Selective Deploy (Tags / Limits) @@ -1242,7 +1241,7 @@ make verify-tailscale > **Note:** Tailscale key rotation does not disrupt existing mesh connections. Nodes already authenticated remain connected. New keys apply only on re-auth. -> **Vault retirement gate (D4):** the Tailscale auth/ACL keys (the "trio": `tailscale_auth_key`, `tailscale_acl_key`, `tailscale_acl_client_id`) are the ONLY remaining Ansible Vault contents. At key expiry: (1) delete the trio from the vault, (2) remove `inventory/group_vars/all/secrets.yml`, (3) remove the `--ask-vault-pass` literals from the six tailscale consumers in the Makefile (`deploy-hardening`, `deploy-compliance`, `reconnect-tailscale`, `deploy-local`, `nuke`, `provision-host`), (4) verify `grep -n -- '--ask-vault-pass' Makefile` returns zero matches - the vault then exists only for bootstrap. +> **Vault retirement gate (D4):** the Tailscale auth/ACL keys (the "trio": `tailscale_auth_key`, `tailscale_acl_key`, `tailscale_acl_client_id`) are the ONLY remaining Ansible Vault contents. At key expiry: (1) delete the trio from the vault, (2) remove `inventory/group_vars/all/secrets.yml`, (3) remove the `--ask-vault-pass` literals from the five tailscale consumers in the Makefile (`deploy-hardening`, `deploy-compliance`, `reconnect-tailscale`, `nuke`, `provision-host`), (4) verify `grep -n -- '--ask-vault-pass' Makefile` returns zero matches - the vault then exists only for bootstrap. ### 7.5 Docker Credential Rotation diff --git a/docs/project/REPOSITORY_STRUCTURE.md b/docs/project/REPOSITORY_STRUCTURE.md index add1545..e63a628 100644 --- a/docs/project/REPOSITORY_STRUCTURE.md +++ b/docs/project/REPOSITORY_STRUCTURE.md @@ -144,8 +144,7 @@ playbooks/ └── ops/ ├── bootstrap.yml # Initial node setup and toolchain install ├── nuke.yml # Deterministic teardown (confirmation-gated) - ├── validate.yml # Syntax and pre-flight validation - └── local-devices.yml # Workstation/laptop hardening over Tailscale + └── validate.yml # Syntax and pre-flight validation ``` ### Playbook Tags @@ -286,7 +285,6 @@ The Makefile is the primary operational interface. All targets run through the ` | `deploy-backup-stack` | Deploy Restic stack backup (brain only) | `playbooks/l6/backup-stack.yml` | | `deploy-backup-engine` | Deploy the unified backup engine (all backup hosts) | `playbooks/l6/backup-engine.yml` | | `deploy-backups` | All backup layers (umbrella: stack → engine) | umbrella target | -| `deploy-local` | Workstation hardening | `playbooks/ops/local-devices.yml` | | `run` | Generic runner (`PLAYBOOK=`, `TAGS=`, `SKIP_TAGS=`, `CHECK=1`) | variable | | `nuke` | Destructive teardown (requires CONFIRM=) | `playbooks/ops/nuke.yml` | | `deploy-platform` | Complete platform (hardening→engine→monitoring→portainer→backups) | meta-target (L4 edge excluded) | diff --git a/docs/security/DEVELOPER_WORKSTATION_GUIDE.md b/docs/security/DEVELOPER_WORKSTATION_GUIDE.md new file mode 100644 index 0000000..c43ae76 --- /dev/null +++ b/docs/security/DEVELOPER_WORKSTATION_GUIDE.md @@ -0,0 +1,98 @@ +# Developer Workstation Security Baseline + +> **Security Standard** - Operator & Developer Endpoint Baseline for `developmi-platform` +> **Model**: Zero Trust (NIST SP 800-207) | **Scope**: macOS, Linux, Windows (WSL2) + +--- + +## 1. Architectural Philosophy: Zero Trust Over Endpoint Push + +In `developmi-platform`, server infrastructure never trusts client workstations by default, and workstations are **not** managed via Ansible push playbooks over SSH. Laptops sleep, roam dynamic networks, and run diverse operating systems where server-grade kernel drops (`policy drop`, aggressive sysctl) break local developer experience (Docker Desktop, local test ports, debuggers). + +Instead, security is enforced through **Zero Trust Network Access (ZTNA)** and **Hardware-Backed Cryptographic Identity**: + +- The server cluster assumes client devices may be untrusted or compromised. +- All access to server control planes and SSH is gated behind **Tailscale mesh ACLs**, MFA, and hardware security keys. +- Developers maintain a standardized local security baseline without orchestration overhead. + +--- + +## 2. The 4 Essential Pillars of the Client Baseline + +``` +┌────────────────────────────────────────────────────────────────────────┐ +│ CLIENT WORKSTATION BASELINE │ +├──────────────────┬──────────────────┬─────────────────┬────────────────┤ +│ 1. Disk Crypto │ 2. FIDO2 / Keys │ 3. Mesh Network │ 4. Secrets │ +│ LUKS / FileVault │ ed25519-sk │ Tailscale + MFA │ SOPS + age │ +│ BitLocker (TPM) │ Signed Commits │ Least Privilege │ 1Password/Bitw │ +└──────────────────┴──────────────────┴─────────────────┴────────────────┘ +``` + +### Pillar 1: Full Disk Encryption (At-Rest Protection) + +Physical theft or loss of a developer device must not compromise local repositories, SSH private keys, or session tokens. + +| Operating System | Encryption Engine | Standard Requirement | +| :----------------- | :------------------- | :------------------------------------------------------------------ | +| **macOS** | **FileVault** | Mandatory enabled via System Settings. Secure Enclave backed. | +| **Linux** | **LUKS2 (dm-crypt)** | Mandatory on `/` and `/home`. Modern distros enable during install. | +| **Windows / WSL2** | **BitLocker** | Mandatory on OS volume using TPM 2.0 with PIN. | + +--- + +### Pillar 2: Hardware-Backed Cryptographic Identity (In-Transit Protection) + +Private keys used for Git commit signing and cluster SSH must **never** be exportable from the machine. + +#### 2.1 SSH Authentication via FIDO2 / Hardware Keys + +Generate a hardware-backed resident key (e.g. YubiKey 5 Series) that requires physical touch: + +```bash +# Generate FIDO2 resident key (requires physical touch on use) +ssh-keygen -t ed25519-sk -O resident -O verify-required -C "operator@developmi" +``` + +_Benefit: Even if malware compromises the laptop filesystem, the private key cannot be extracted without the physical token._ + +#### 2.2 Cryptographic Git Commit Signing + +All commits to `developmi-platform` require GPG or SSH signature verification (`git commit -S`). + +Configure SSH-based commit signing natively in Git: + +```bash +git config --global gpg.format ssh +git config --global user.signingkey ~/.ssh/id_ed25519_sk.pub +git config --global commit.gpgsign true +``` + +--- + +### Pillar 3: Overlay Network & Least Privilege (Tailscale) + +Client devices connect to the cluster strictly as client nodes on the private Tailnet: + +1. **No Inbound Public Ports**: Client workstations must not listen on public IPv4/IPv6 interfaces. +2. **Tailscale Multi-Factor Authentication (MFA)**: Operators authenticate into Tailscale using a corporate IdP or GitHub OAuth with WebAuthn/FIDO2 MFA. +3. **ACL Enforcement**: The Tailscale ACL policy ([roles/L2_compliance/general/templates/tailscale/tailscale-acls.json.j2](file:///mnt/Workspace/_03_INFRA/developmi-platform/roles/L2_compliance/general/templates/tailscale/tailscale-acls.json.j2)) restricts operator traffic strictly to authorized management ports (`tag:brain:22`, `tag:muscle:22`, `tag:muscle:8443`). + +--- + +### Pillar 4: Secret Management & Hygiene + +- **SOPS + age**: All environment secrets and keys in this repo reside encrypted in `inventory/group_vars/all/secrets.sops.yml`. +- **Private age Key Storage**: The operator's age private key (`~/.config/sops/age/keys.txt`) must have file mode `0600` and be stored in a password manager (e.g. 1Password / Bitwarden). +- **Zero Plaintext Secrets in Dotfiles**: Do not commit secrets, tokens, or age keys to personal dotfile repositories. + +--- + +## 3. Quick Checklist for New Developers / Operators + +- [ ] Full disk encryption enabled and verified on laptop. +- [ ] OS automatic security patches enabled. +- [ ] Tailscale client installed and enrolled in the cluster Tailnet with MFA. +- [ ] Hardware key (YubiKey) or hardware-backed SSH key configured for Git signing and SSH. +- [ ] Pre-commit hooks installed locally: `make precommit-install`. +- [ ] Age key imported: `mkdir -p ~/.config/sops/age && chmod 700 ~/.config/sops/age`. diff --git a/docs/security/HARDENING-STATUS.md b/docs/security/HARDENING-STATUS.md index 5c1d32e..ba14af0 100644 --- a/docs/security/HARDENING-STATUS.md +++ b/docs/security/HARDENING-STATUS.md @@ -20,14 +20,13 @@ what requires manual intervention, and what is not applicable per host class. ## Hardening Profile Dispatcher The `hardening_profile` variable (from [ARCHITECTURE.md §3](../architecture/ARCHITECTURE.md#3-role-composition-and-variable-hierarchy)) -dispatches between two hardening levels: +sets the server-grade hardening baseline: -| Profile | Value | Host Classes | Behavior | -| --------------- | ------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Server** | `server` | brain, muscle | Full hardening: server-grade firewall (`ufw deny`/`nftables deny`), fail2ban 1h bantime, CrowdSec enforcement, observability stack, backups enabled | -| **Workstation** | `workstation` | local | Tailored hardening: UFW deny (Zero Trust base), fail2ban 10m bantime, CrowdSec skipped (false positives from network changes), no observability, no backups | +| Profile | Value | Host Classes | Behavior | +| ---------- | -------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Server** | `server` | brain, muscle | Full hardening: in-kernel nftables firewall (`policy drop`), fail2ban 1h bantime, CrowdSec enforcement, observability stack, and Restic backups enabled | -The dispatcher is set per host class in `inventory/group_vars/{brain,muscle,local}/main.yml`. +The dispatcher is set per host class in `inventory/group_vars/{brain,muscle}/main.yml`. Developer and operator workstation security follows a Zero Trust baseline outside Ansible management (see [DEVELOPER_WORKSTATION_GUIDE.md](DEVELOPER_WORKSTATION_GUIDE.md)). ### Tag Conventions Used in Matrix @@ -90,17 +89,6 @@ Status values: | L5 | App Profiles | N/A | - | Reference profiles at `apps/` (user-managed) | | L6 | Runtime | Automated | `playbooks/l6/engine.yml`, `playbooks/l6/portainer.yml`, `playbooks/l6/backup-stack.yml`, `playbooks/l6/backup-appdata.yml`, `playbooks/l6/backup-timers.yml` | `l6-runtime`, Docker Engine, Portainer (optional, Edge Agent), stack backup, app data backup, Docker bench | -### Local - -| Layer | Hardening | Status | Playbook | Tags / Notes | -| ----- | -------------------- | --------- | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | -| L1 | OS baseline | Automated | `playbooks/l1/baseline.yml` | `l1-os-baseline`, CIS Level 1, kernel tuning | -| L2 | Compliance hardening | Automated | `playbooks/l2/hardening.yml`, `playbooks/l2/compliance.yml` | `l2-compliance`, `hardening_profile: workstation`, `nist,sc-7`, SSH, firewall, fail2ban (10m), kernel. CrowdSec **skipped** | -| L3 | Observability | N/A | - | No observability on workstation class | -| L4 | Networking / Edge | N/A | - | No ingress role on local | -| L5 | App Profiles | N/A | - | No app profiles on workstation class | -| L6 | Runtime | Automated | `playbooks/l6/engine.yml` | `l6-runtime`, Docker Engine only (no Portainer, no backup) | - --- ## Automation Level Summary @@ -109,7 +97,6 @@ Status values: | ------ | --------- | --------- | --------- | --------- | --- | --------- | | Brain | Automated | Automated | Automated | N/A | N/A | Automated | | Muscle | Automated | Automated | Automated | Automated | N/A | Automated | -| Local | Automated | Automated | N/A | N/A | N/A | Automated | --- @@ -119,14 +106,14 @@ NIST 800-53 controls are tagged at the Ansible task level. Evidence is auto-coll to `/srv/evidence/nist//`. See [NIST_800_53.md](../compliance/NIST/NIST_800_53.md) for full control-to-task mapping. -| Control | Tag | Layer | Applies To | Enforcement | -| ------- | ------------ | ----- | -------------------- | ----------------------------------------- | -| AC-2 | `nist,ac-2` | L2 | brain, muscle | Automated (account management) | -| SC-7 | `nist,sc-7` | L2 | brain, muscle, local | Automated (boundary protection) | -| SI-4 | `nist,si-4` | L2 | brain, muscle | Automated (monitoring) | -| AU-12 | `nist,au-12` | L2 | brain, muscle | Automated (audit generation) | -| CM-7 | `nist,cm-7` | L2 | brain, muscle | Automated (least functionality) | -| SC-28 | `nist,sc-28` | L2 | brain, muscle | Audit-only (disk encryption verification) | +| Control | Tag | Layer | Applies To | Enforcement | +| ------- | ------------ | ----- | ------------- | ----------------------------------------- | +| AC-2 | `nist,ac-2` | L2 | brain, muscle | Automated (account management) | +| SC-7 | `nist,sc-7` | L2 | brain, muscle | Automated (boundary protection) | +| SI-4 | `nist,si-4` | L2 | brain, muscle | Automated (monitoring) | +| AU-12 | `nist,au-12` | L2 | brain, muscle | Automated (audit generation) | +| CM-7 | `nist,cm-7` | L2 | brain, muscle | Automated (least functionality) | +| SC-28 | `nist,sc-28` | L2 | brain, muscle | Audit-only (disk encryption verification) | ---