From 4bbab88a2fae8f4f3f963d20b8594991bb72a106 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Mon, 10 Aug 2026 15:20:03 +0000 Subject: [PATCH 01/24] add chrony cluster-wide ntp server --- fleet/vaclab-ntp/config/config.yaml | 17 +++++++++ fleet/vaclab-ntp/config/fleet.yaml | 1 + fleet/vaclab-ntp/config/volume.yaml | 11 ++++++ fleet/vaclab-ntp/fleet.yaml | 4 ++ fleet/vaclab-ntp/server/fleet.yaml | 3 ++ fleet/vaclab-ntp/server/ntp-server.yaml | 51 +++++++++++++++++++++++++ 6 files changed, 87 insertions(+) create mode 100644 fleet/vaclab-ntp/config/config.yaml create mode 100644 fleet/vaclab-ntp/config/fleet.yaml create mode 100644 fleet/vaclab-ntp/config/volume.yaml create mode 100644 fleet/vaclab-ntp/fleet.yaml create mode 100644 fleet/vaclab-ntp/server/fleet.yaml create mode 100644 fleet/vaclab-ntp/server/ntp-server.yaml diff --git a/fleet/vaclab-ntp/config/config.yaml b/fleet/vaclab-ntp/config/config.yaml new file mode 100644 index 0000000..6459a71 --- /dev/null +++ b/fleet/vaclab-ntp/config/config.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vaclab-ntp-server-conf +data: + chrony.conf: | + server consolidare-continuum.com iburst + server time.cloudflare.com iburst + driftfile /var/lib/chrony/drift + rtcsync + makestep 1.0 3 + local stratum 10 + allow 185.181.228.0/24 + allow 65.21.76.50/32 + allow 10.42.0.0/16 + allow 10.43.0.0/16 + logdir /var/log/chrony \ No newline at end of file diff --git a/fleet/vaclab-ntp/config/fleet.yaml b/fleet/vaclab-ntp/config/fleet.yaml new file mode 100644 index 0000000..404da2b --- /dev/null +++ b/fleet/vaclab-ntp/config/fleet.yaml @@ -0,0 +1 @@ +defaultNamespace: vaclab-ntp \ No newline at end of file diff --git a/fleet/vaclab-ntp/config/volume.yaml b/fleet/vaclab-ntp/config/volume.yaml new file mode 100644 index 0000000..105b874 --- /dev/null +++ b/fleet/vaclab-ntp/config/volume.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ntp-drift-pvc +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn-vaclab + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/fleet/vaclab-ntp/fleet.yaml b/fleet/vaclab-ntp/fleet.yaml new file mode 100644 index 0000000..52b8dcc --- /dev/null +++ b/fleet/vaclab-ntp/fleet.yaml @@ -0,0 +1,4 @@ +defaultNamespace: vaclab-ntp +dependsOn: + - name: vaclab-2-fleet-longhorn + - name: vaclab-2-fleet-storage \ No newline at end of file diff --git a/fleet/vaclab-ntp/server/fleet.yaml b/fleet/vaclab-ntp/server/fleet.yaml new file mode 100644 index 0000000..771e48d --- /dev/null +++ b/fleet/vaclab-ntp/server/fleet.yaml @@ -0,0 +1,3 @@ +defaultNamespace: vaclab-ntp +dependsOn: + - name: vaclab-2-fleet-vaclab-ntp-config \ No newline at end of file diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml new file mode 100644 index 0000000..fd59a3c --- /dev/null +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ntp-server +spec: + replicas: 2 + selector: + matchLabels: + app: ntp-server + template: + metadata: + labels: + app: ntp-server + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - node-01.ih-eu-mda1.misc.vaclab.status.im + - metal-01.he-eu-hel1.misc.vacdst.status.im + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ntp-server + topologyKey: kubernetes.io/hostname + containers: + - name: ntp-server + image: cturra/ntp:latest + securityContext: + capabilities: + add: ["SYS_TIME", "NET_BIND_SERVICE"] + volumeMounts: + - name: conf + mountPath: /etc/chrony/chrony.conf + subPath: chrony.conf + - name: ntp-drift + mountPath: /var/lib/chrony + volumes: + - name: conf + configMap: + name: vaclab-ntp-server-conf + - name: ntp-drift + persistentVolumeClaim: + claimName: ntp-drift-pvc \ No newline at end of file From 6a6cec631761f7efcd1ae2df244401f5a804b446 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Mon, 10 Aug 2026 15:27:05 +0000 Subject: [PATCH 02/24] fix node name --- fleet/vaclab-ntp/server/ntp-server.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index fd59a3c..952e47f 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -22,8 +22,8 @@ spec: - key: kubernetes.io/hostname operator: In values: - - node-01.ih-eu-mda1.misc.vaclab.status.im - - metal-01.he-eu-hel1.misc.vacdst.status.im + - node-01.ih-eu-mda1.misc.vaclab + - metal-01.he-eu-hel1.misc.vacdst podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: From dde5314821b63475b7571592f2b06acf2e90e120 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Mon, 10 Aug 2026 15:30:01 +0000 Subject: [PATCH 03/24] fix nft volume issues --- fleet/vaclab-ntp/config/volume.yaml | 11 ----------- fleet/vaclab-ntp/server/ntp-server.yaml | 13 ++++++++++--- 2 files changed, 10 insertions(+), 14 deletions(-) delete mode 100644 fleet/vaclab-ntp/config/volume.yaml diff --git a/fleet/vaclab-ntp/config/volume.yaml b/fleet/vaclab-ntp/config/volume.yaml deleted file mode 100644 index 105b874..0000000 --- a/fleet/vaclab-ntp/config/volume.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ntp-drift-pvc -spec: - accessModes: - - ReadWriteOnce - storageClassName: longhorn-vaclab - resources: - requests: - storage: 1Gi \ No newline at end of file diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index 952e47f..33268a5 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -46,6 +46,13 @@ spec: - name: conf configMap: name: vaclab-ntp-server-conf - - name: ntp-drift - persistentVolumeClaim: - claimName: ntp-drift-pvc \ No newline at end of file + volumeClaimTemplates: + - metadata: + name: ntp-drift + spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn-vaclab + resources: + requests: + storage: 1Gi \ No newline at end of file From 65565ac7b5b84ef00f99e292bfdcdeacc5dcb283 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Mon, 10 Aug 2026 17:05:29 +0000 Subject: [PATCH 04/24] prefer cloudflare ntp server --- fleet/vaclab-ntp/config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fleet/vaclab-ntp/config/config.yaml b/fleet/vaclab-ntp/config/config.yaml index 6459a71..2cdfdf4 100644 --- a/fleet/vaclab-ntp/config/config.yaml +++ b/fleet/vaclab-ntp/config/config.yaml @@ -4,8 +4,8 @@ metadata: name: vaclab-ntp-server-conf data: chrony.conf: | + server time.cloudflare.com iburst prefer server consolidare-continuum.com iburst - server time.cloudflare.com iburst driftfile /var/lib/chrony/drift rtcsync makestep 1.0 3 From c0c1f96da7ba9c7a1c3d706c3637e65626e81264 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Mon, 10 Aug 2026 17:21:35 +0000 Subject: [PATCH 05/24] do cpu pinning for ntp server --- fleet/vaclab-ntp/server/ntp-server.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index 33268a5..4680490 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -32,6 +32,13 @@ spec: topologyKey: kubernetes.io/hostname containers: - name: ntp-server + resources: + requests: + memory: "1Gi" + cpu: "1" + limits: + memory: "1Gi" + cpu: "1" image: cturra/ntp:latest securityContext: capabilities: From 6a67cb10848fd82f4280af5f82e534da0192dd3f Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Mon, 10 Aug 2026 17:48:34 +0000 Subject: [PATCH 06/24] update ansible --- ansible/playbooks/tasks/chrony_client.yaml | 29 +++++++++++++++++++++ ansible/playbooks/update_chrony_client.yaml | 26 ++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 ansible/playbooks/tasks/chrony_client.yaml create mode 100644 ansible/playbooks/update_chrony_client.yaml diff --git a/ansible/playbooks/tasks/chrony_client.yaml b/ansible/playbooks/tasks/chrony_client.yaml new file mode 100644 index 0000000..ad21264 --- /dev/null +++ b/ansible/playbooks/tasks/chrony_client.yaml @@ -0,0 +1,29 @@ +--- +- name: Install chrony + ansible.builtin.apt: + name: chrony + state: present + update_cache: true + +- name: Configure chrony to sync against the internal vaclab-ntp servers + ansible.builtin.copy: + dest: /etc/chrony/chrony.conf + mode: '0644' + content: | + # Managed by Ansible: update_chrony_client.yaml + # IPs (not hostnames) to avoid depending on DNS for time sync. + # node-01.ih-eu-mda1.misc.vaclab + server 185.181.228.122 iburst + # metal-01.he-eu-hel1.misc.vacdst + server 65.21.76.50 iburst + driftfile /var/lib/chrony/chrony.drift + rtcsync + makestep 1.0 3 + logdir /var/log/chrony + notify: Restart chrony + +- name: Ensure chrony service is enabled and running + ansible.builtin.systemd: + name: chrony + state: started + enabled: true diff --git a/ansible/playbooks/update_chrony_client.yaml b/ansible/playbooks/update_chrony_client.yaml new file mode 100644 index 0000000..9aa8d83 --- /dev/null +++ b/ansible/playbooks/update_chrony_client.yaml @@ -0,0 +1,26 @@ +--- +- name: Configure chrony NTP client on cluster hosts + hosts: k3s_cluster + become: true + handlers: + - name: Restart chrony + ansible.builtin.systemd: + name: chrony + state: restarted + + tasks: + - name: Apply chrony client configuration + ansible.builtin.import_tasks: tasks/chrony_client.yaml + +- name: Verify chrony sync status + hosts: k3s_cluster + become: true + tasks: + - name: Check chrony sources + ansible.builtin.command: chronyc sources + register: chrony_sources + changed_when: false + + - name: Display chrony sources + ansible.builtin.debug: + var: chrony_sources.stdout_lines From 3f79a30cd0b4d1604a95ddcdec50af30d1e9bcd4 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Mon, 10 Aug 2026 17:53:39 +0000 Subject: [PATCH 07/24] update ansible --- ansible/playbooks/tasks/chrony_client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/tasks/chrony_client.yaml b/ansible/playbooks/tasks/chrony_client.yaml index ad21264..2956a2f 100644 --- a/ansible/playbooks/tasks/chrony_client.yaml +++ b/ansible/playbooks/tasks/chrony_client.yaml @@ -13,7 +13,7 @@ # Managed by Ansible: update_chrony_client.yaml # IPs (not hostnames) to avoid depending on DNS for time sync. # node-01.ih-eu-mda1.misc.vaclab - server 185.181.228.122 iburst + server 185.181.228.122 iburst prefer # metal-01.he-eu-hel1.misc.vacdst server 65.21.76.50 iburst driftfile /var/lib/chrony/chrony.drift From 0afa87f005b39f62fdec910802e8df1267b4a92a Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Mon, 10 Aug 2026 19:17:41 +0000 Subject: [PATCH 08/24] update ansible with ntp report --- ansible/playbooks/chrony_report.yaml | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 ansible/playbooks/chrony_report.yaml diff --git a/ansible/playbooks/chrony_report.yaml b/ansible/playbooks/chrony_report.yaml new file mode 100644 index 0000000..388e666 --- /dev/null +++ b/ansible/playbooks/chrony_report.yaml @@ -0,0 +1,36 @@ +--- +- name: Gather chrony sync status from all cluster nodes + hosts: k3s_cluster + become: true + gather_facts: false + tasks: + - name: Get chrony tracking summary + ansible.builtin.shell: | + set -o pipefail + chronyc tracking | awk -F': ' ' + /^Stratum/ {stratum=$2} + /^System time/ {systime=$2} + /^Last offset/ {lastoff=$2} + /^RMS offset/ {rms=$2} + /^Root dispersion/ {rootdisp=$2} + /^Leap status/ {leap=$2} + END {printf "%-8s %-32s %-18s %-18s %-18s %-12s", stratum, systime, lastoff, rms, rootdisp, leap} + ' + args: + executable: /bin/bash + register: chrony_summary + changed_when: false + +- name: Display cluster-wide chrony sync table + hosts: server[0] + become: false + gather_facts: false + tasks: + - name: Print table header + ansible.builtin.debug: + msg: "{{ '%-42s' | format('HOST') }} {{ '%-8s %-32s %-18s %-18s %-18s %-12s' | format('STRATUM', 'SYSTEM TIME OFFSET', 'LAST OFFSET', 'RMS OFFSET', 'ROOT DISPERSION', 'LEAP STATUS') }}" + + - name: Print each node's row + ansible.builtin.debug: + msg: "{{ '%-42s' | format(item) }} {{ hostvars[item].chrony_summary.stdout }}" + loop: "{{ groups['k3s_cluster'] }}" From 9468a2479c6d17db0c6eab4b094eec2b6699800a Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Mon, 10 Aug 2026 19:22:57 +0000 Subject: [PATCH 09/24] update ansible with ntp report --- ansible/playbooks/chrony_report.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/chrony_report.yaml b/ansible/playbooks/chrony_report.yaml index 388e666..4fb460b 100644 --- a/ansible/playbooks/chrony_report.yaml +++ b/ansible/playbooks/chrony_report.yaml @@ -9,12 +9,12 @@ set -o pipefail chronyc tracking | awk -F': ' ' /^Stratum/ {stratum=$2} - /^System time/ {systime=$2} + /^System time/ {sys_time=$2} /^Last offset/ {lastoff=$2} /^RMS offset/ {rms=$2} /^Root dispersion/ {rootdisp=$2} /^Leap status/ {leap=$2} - END {printf "%-8s %-32s %-18s %-18s %-18s %-12s", stratum, systime, lastoff, rms, rootdisp, leap} + END {printf "%-8s %-32s %-18s %-18s %-18s %-12s", stratum, sys_time, lastoff, rms, rootdisp, leap} ' args: executable: /bin/bash From 79bcd0d4e22fea22161994b3def54a5efae5e61e Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Tue, 11 Aug 2026 06:17:14 +0000 Subject: [PATCH 10/24] use consolidare-continuum.com as the main ntp choice --- fleet/vaclab-ntp/config/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fleet/vaclab-ntp/config/config.yaml b/fleet/vaclab-ntp/config/config.yaml index 2cdfdf4..1520a05 100644 --- a/fleet/vaclab-ntp/config/config.yaml +++ b/fleet/vaclab-ntp/config/config.yaml @@ -4,8 +4,8 @@ metadata: name: vaclab-ntp-server-conf data: chrony.conf: | - server time.cloudflare.com iburst prefer - server consolidare-continuum.com iburst + server consolidare-continuum.com iburst prefer + server time.cloudflare.com iburst driftfile /var/lib/chrony/drift rtcsync makestep 1.0 3 From 97d939544f75c3d7af0e7005ad6263df3ebf5142 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Tue, 11 Aug 2026 06:41:33 +0000 Subject: [PATCH 11/24] use metal-01 as the sole server --- ansible/inventory.yaml | 8 ++++++ ansible/playbooks/tasks/chrony_client.yaml | 12 ++++----- ansible/playbooks/update_chrony_client.yaml | 27 ++++++++++++++++++--- fleet/vaclab-ntp/server/ntp-server.yaml | 21 ++++++---------- 4 files changed, 45 insertions(+), 23 deletions(-) diff --git a/ansible/inventory.yaml b/ansible/inventory.yaml index 1dfa0c8..0e55a4b 100644 --- a/ansible/inventory.yaml +++ b/ansible/inventory.yaml @@ -1,4 +1,12 @@ --- +# Hosts that run a vaclab-ntp server pod (hostNetwork) locally. Their system +# clock is already disciplined directly by that pod's chronyd, so they must +# NOT also run a native host-level chrony client - two chronyd instances +# steering the same kernel clock fight each other and cause large offsets. +ntp_server_hosts: + hosts: + metal-01.he-eu-hel1.misc.vacdst.status.im: + k3s_cluster: children: server: diff --git a/ansible/playbooks/tasks/chrony_client.yaml b/ansible/playbooks/tasks/chrony_client.yaml index 2956a2f..f4cc3a0 100644 --- a/ansible/playbooks/tasks/chrony_client.yaml +++ b/ansible/playbooks/tasks/chrony_client.yaml @@ -5,17 +5,17 @@ state: present update_cache: true -- name: Configure chrony to sync against the internal vaclab-ntp servers +- name: Configure chrony to sync against the internal vaclab-ntp server, with Cloudflare as backup ansible.builtin.copy: dest: /etc/chrony/chrony.conf mode: '0644' content: | # Managed by Ansible: update_chrony_client.yaml - # IPs (not hostnames) to avoid depending on DNS for time sync. - # node-01.ih-eu-mda1.misc.vaclab - server 185.181.228.122 iburst prefer - # metal-01.he-eu-hel1.misc.vacdst - server 65.21.76.50 iburst + # Primary: the sole vaclab-ntp server (metal-01.he-eu-hel1.misc.vacdst). + # IP (not hostname) to avoid depending on DNS for time sync. + server 65.21.76.50 iburst prefer + # Backup: only used if metal-01 becomes unreachable. + server time.cloudflare.com iburst driftfile /var/lib/chrony/chrony.drift rtcsync makestep 1.0 3 diff --git a/ansible/playbooks/update_chrony_client.yaml b/ansible/playbooks/update_chrony_client.yaml index 9aa8d83..61301bb 100644 --- a/ansible/playbooks/update_chrony_client.yaml +++ b/ansible/playbooks/update_chrony_client.yaml @@ -1,6 +1,26 @@ --- -- name: Configure chrony NTP client on cluster hosts - hosts: k3s_cluster +- name: Remove conflicting host-level chrony on vaclab-ntp server hosts + hosts: ntp_server_hosts + become: true + tasks: + # These hosts run a vaclab-ntp server pod with hostNetwork: true, whose + # chronyd already disciplines this host's kernel clock directly. A + # second, independent chronyd here would fight it for control of the + # same clock (confirmed: both ended up bound to 127.0.0.1:323 at once). + - name: Stop and disable native chrony service + ansible.builtin.systemd: + name: chrony + state: stopped + enabled: false + ignore_errors: true + + - name: Remove chrony package + ansible.builtin.apt: + name: chrony + state: absent + +- name: Configure chrony NTP client on cluster hosts without a local ntp-server pod + hosts: k3s_cluster:!ntp_server_hosts become: true handlers: - name: Restart chrony @@ -16,10 +36,11 @@ hosts: k3s_cluster become: true tasks: - - name: Check chrony sources + - name: Check chrony sources (or note that this host has no host-level chrony) ansible.builtin.command: chronyc sources register: chrony_sources changed_when: false + failed_when: false - name: Display chrony sources ansible.builtin.debug: diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index 4680490..87ece1e 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -3,7 +3,7 @@ kind: StatefulSet metadata: name: ntp-server spec: - replicas: 2 + replicas: 1 selector: matchLabels: app: ntp-server @@ -22,23 +22,16 @@ spec: - key: kubernetes.io/hostname operator: In values: - - node-01.ih-eu-mda1.misc.vaclab - metal-01.he-eu-hel1.misc.vacdst - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: ntp-server - topologyKey: kubernetes.io/hostname containers: - name: ntp-server resources: - requests: - memory: "1Gi" - cpu: "1" - limits: - memory: "1Gi" - cpu: "1" + requests: + memory: "1Gi" + cpu: "1" + limits: + memory: "1Gi" + cpu: "1" image: cturra/ntp:latest securityContext: capabilities: From bfde236106a8bdcf179a232752c7fbc070e7d262 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Tue, 11 Aug 2026 06:47:02 +0000 Subject: [PATCH 12/24] use metal-01 as the sole server --- ansible/playbooks/chrony_report.yaml | 52 ++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/ansible/playbooks/chrony_report.yaml b/ansible/playbooks/chrony_report.yaml index 4fb460b..895b81b 100644 --- a/ansible/playbooks/chrony_report.yaml +++ b/ansible/playbooks/chrony_report.yaml @@ -1,10 +1,10 @@ --- -- name: Gather chrony sync status from all cluster nodes - hosts: k3s_cluster +- name: Gather chrony tracking status from client hosts + hosts: k3s_cluster:!ntp_server_hosts become: true gather_facts: false tasks: - - name: Get chrony tracking summary + - name: Get chrony tracking summary (host-level) ansible.builtin.shell: | set -o pipefail chronyc tracking | awk -F': ' ' @@ -21,16 +21,48 @@ register: chrony_summary changed_when: false -- name: Display cluster-wide chrony sync table +# ntp_server_hosts (currently just metal-01) have no host-level chronyc +# anymore - only the co-located ntp-server pod (replicas: 1, ordinal 0) +# disciplines that host's clock. Query the pod itself instead. +- name: Gather chrony tracking status from the vaclab-ntp server pod hosts: server[0] become: false gather_facts: false + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml tasks: - - name: Print table header - ansible.builtin.debug: - msg: "{{ '%-42s' | format('HOST') }} {{ '%-8s %-32s %-18s %-18s %-18s %-12s' | format('STRATUM', 'SYSTEM TIME OFFSET', 'LAST OFFSET', 'RMS OFFSET', 'ROOT DISPERSION', 'LEAP STATUS') }}" + - name: Get chrony tracking summary from ntp-server-0 + ansible.builtin.shell: | + set -o pipefail + kubectl exec -n vaclab-ntp ntp-server-0 -- chronyc tracking | awk -F': ' ' + /^Stratum/ {stratum=$2} + /^System time/ {sys_time=$2} + /^Last offset/ {lastoff=$2} + /^RMS offset/ {rms=$2} + /^Root dispersion/ {rootdisp=$2} + /^Leap status/ {leap=$2} + END {printf "%-8s %-32s %-18s %-18s %-18s %-12s", stratum, sys_time, lastoff, rms, rootdisp, leap} + ' + args: + executable: /bin/bash + register: chrony_summary_ntp_server_pod + changed_when: false - - name: Print each node's row +- name: Display cluster-wide chrony sync table + hosts: server[0] + become: false + gather_facts: false + tasks: + - name: Print combined table + vars: + header: "{{ '%-44s' | format('HOST') }} {{ '%-8s %-32s %-18s %-18s %-18s %-12s' | format('STRATUM', 'SYSTEM TIME OFFSET', 'LAST OFFSET', 'RMS OFFSET', 'ROOT DISPERSION', 'LEAP STATUS') }}" ansible.builtin.debug: - msg: "{{ '%-42s' | format(item) }} {{ hostvars[item].chrony_summary.stdout }}" - loop: "{{ groups['k3s_cluster'] }}" + msg: | + {{ header }} + {% for item in groups['k3s_cluster'] %} + {% if item in groups['ntp_server_hosts'] %} + {{ '%-44s' | format(item ~ ' (via pod)') }} {{ chrony_summary_ntp_server_pod.stdout }} + {% else %} + {{ '%-44s' | format(item) }} {{ hostvars[item].chrony_summary.stdout }} + {% endif %} + {% endfor %} From 420b3c1be8842bdd3ab8d6da4d7c827903aab7d4 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Tue, 11 Aug 2026 07:01:19 +0000 Subject: [PATCH 13/24] deop cpu pinning for ntp server --- fleet/vaclab-ntp/server/ntp-server.yaml | 11 +++++++++-- fleet/vaclab-ntp/server/service.yaml | 8 ++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 fleet/vaclab-ntp/server/service.yaml diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index 87ece1e..a9d5478 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -4,6 +4,7 @@ metadata: name: ntp-server spec: replicas: 1 + serviceName: ntp-server selector: matchLabels: app: ntp-server @@ -26,9 +27,15 @@ spec: containers: - name: ntp-server resources: + # requests.cpu != limits.cpu on purpose: this must NOT be + # Guaranteed QoS. A Guaranteed pod gets an exclusive, pinned + # physical core under the cluster's static CPU manager policy, + # which is suspected of causing this pod's chronyd to measure a + # large, still-growing clock frequency error (~500ppm) despite + # otherwise clean, low-noise measurements. requests: - memory: "1Gi" - cpu: "1" + memory: "512Mi" + cpu: "250m" limits: memory: "1Gi" cpu: "1" diff --git a/fleet/vaclab-ntp/server/service.yaml b/fleet/vaclab-ntp/server/service.yaml new file mode 100644 index 0000000..5f67989 --- /dev/null +++ b/fleet/vaclab-ntp/server/service.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Service +metadata: + name: ntp-server +spec: + clusterIP: None + selector: + app: ntp-server From 617990b390e1427de3cce8c69a832154d092e164 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Tue, 11 Aug 2026 07:38:04 +0000 Subject: [PATCH 14/24] drop hostNetwork for ntp server --- ansible/inventory.yaml | 8 --- ansible/playbooks/chrony_report.yaml | 66 +++++++------------ ansible/playbooks/tasks/chrony_client.yaml | 10 +-- ansible/playbooks/update_chrony_client.yaml | 27 +------- fleet/vaclab-ntp/config/config.yaml | 2 +- fleet/vaclab-ntp/server/ntp-server.yaml | 35 ++++------ fleet/vaclab-ntp/server/service-nodeport.yaml | 14 ++++ 7 files changed, 59 insertions(+), 103 deletions(-) create mode 100644 fleet/vaclab-ntp/server/service-nodeport.yaml diff --git a/ansible/inventory.yaml b/ansible/inventory.yaml index 0e55a4b..1dfa0c8 100644 --- a/ansible/inventory.yaml +++ b/ansible/inventory.yaml @@ -1,12 +1,4 @@ --- -# Hosts that run a vaclab-ntp server pod (hostNetwork) locally. Their system -# clock is already disciplined directly by that pod's chronyd, so they must -# NOT also run a native host-level chrony client - two chronyd instances -# steering the same kernel clock fight each other and cause large offsets. -ntp_server_hosts: - hosts: - metal-01.he-eu-hel1.misc.vacdst.status.im: - k3s_cluster: children: server: diff --git a/ansible/playbooks/chrony_report.yaml b/ansible/playbooks/chrony_report.yaml index 895b81b..aaf63e5 100644 --- a/ansible/playbooks/chrony_report.yaml +++ b/ansible/playbooks/chrony_report.yaml @@ -1,53 +1,36 @@ --- -- name: Gather chrony tracking status from client hosts - hosts: k3s_cluster:!ntp_server_hosts +- name: Gather chrony sync status from all cluster nodes + hosts: k3s_cluster become: true gather_facts: false tasks: - - name: Get chrony tracking summary (host-level) + - name: Get chrony tracking summary ansible.builtin.shell: | set -o pipefail chronyc tracking | awk -F': ' ' - /^Stratum/ {stratum=$2} - /^System time/ {sys_time=$2} - /^Last offset/ {lastoff=$2} - /^RMS offset/ {rms=$2} - /^Root dispersion/ {rootdisp=$2} - /^Leap status/ {leap=$2} - END {printf "%-8s %-32s %-18s %-18s %-18s %-12s", stratum, sys_time, lastoff, rms, rootdisp, leap} + /^Stratum/ {stratum=$2} + /^System time/ { + split($2, a, " "); v = a[1] + 0 + if ($2 ~ /fast/) v = -v + sys_ms = v * 1000 + } + /^Last offset/ { + split($2, a, " "); lastoff_ms = (a[1] + 0) * 1000 + } + /^RMS offset/ { + split($2, a, " "); rms_ms = (a[1] + 0) * 1000 + } + /^Root dispersion/ { + split($2, a, " "); rootdisp_ms = (a[1] + 0) * 1000 + } + /^Leap status/ {leap=$2} + END {printf "%s,%.3f,%.3f,%.3f,%.3f,%s", stratum, sys_ms, lastoff_ms, rms_ms, rootdisp_ms, leap} ' args: executable: /bin/bash register: chrony_summary changed_when: false -# ntp_server_hosts (currently just metal-01) have no host-level chronyc -# anymore - only the co-located ntp-server pod (replicas: 1, ordinal 0) -# disciplines that host's clock. Query the pod itself instead. -- name: Gather chrony tracking status from the vaclab-ntp server pod - hosts: server[0] - become: false - gather_facts: false - environment: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml - tasks: - - name: Get chrony tracking summary from ntp-server-0 - ansible.builtin.shell: | - set -o pipefail - kubectl exec -n vaclab-ntp ntp-server-0 -- chronyc tracking | awk -F': ' ' - /^Stratum/ {stratum=$2} - /^System time/ {sys_time=$2} - /^Last offset/ {lastoff=$2} - /^RMS offset/ {rms=$2} - /^Root dispersion/ {rootdisp=$2} - /^Leap status/ {leap=$2} - END {printf "%-8s %-32s %-18s %-18s %-18s %-12s", stratum, sys_time, lastoff, rms, rootdisp, leap} - ' - args: - executable: /bin/bash - register: chrony_summary_ntp_server_pod - changed_when: false - - name: Display cluster-wide chrony sync table hosts: server[0] become: false @@ -55,14 +38,11 @@ tasks: - name: Print combined table vars: - header: "{{ '%-44s' | format('HOST') }} {{ '%-8s %-32s %-18s %-18s %-18s %-12s' | format('STRATUM', 'SYSTEM TIME OFFSET', 'LAST OFFSET', 'RMS OFFSET', 'ROOT DISPERSION', 'LEAP STATUS') }}" + header: "{{ '%-32s %-8s %12s %14s %10s %14s %-8s' | format('HOST', 'STRATUM', 'OFFSET(ms)', 'LASTOFF(ms)', 'RMS(ms)', 'ROOTDISP(ms)', 'LEAP') }}" ansible.builtin.debug: msg: | {{ header }} {% for item in groups['k3s_cluster'] %} - {% if item in groups['ntp_server_hosts'] %} - {{ '%-44s' | format(item ~ ' (via pod)') }} {{ chrony_summary_ntp_server_pod.stdout }} - {% else %} - {{ '%-44s' | format(item) }} {{ hostvars[item].chrony_summary.stdout }} - {% endif %} + {% set f = hostvars[item].chrony_summary.stdout.split(',') %} + {{ '%-32s %-8s %12s %14s %10s %14s %-8s' | format(item | regex_replace('\.status\.im$', ''), f[0], f[1], f[2], f[3], f[4], f[5]) }} {% endfor %} diff --git a/ansible/playbooks/tasks/chrony_client.yaml b/ansible/playbooks/tasks/chrony_client.yaml index f4cc3a0..6c832fb 100644 --- a/ansible/playbooks/tasks/chrony_client.yaml +++ b/ansible/playbooks/tasks/chrony_client.yaml @@ -11,10 +11,12 @@ mode: '0644' content: | # Managed by Ansible: update_chrony_client.yaml - # Primary: the sole vaclab-ntp server (metal-01.he-eu-hel1.misc.vacdst). - # IP (not hostname) to avoid depending on DNS for time sync. - server 65.21.76.50 iburst prefer - # Backup: only used if metal-01 becomes unreachable. + # Primary: vaclab-ntp server pod, reached via its NodePort (30123/udp) + # on metal-01.he-eu-hel1.misc.vacdst. NodePort routes correctly to the + # pod regardless of which node it's actually scheduled on. IP (not + # hostname) to avoid depending on DNS for time sync. + server 65.21.76.50 port 30123 iburst prefer + # Backup: only used if the vaclab-ntp server is unreachable. server time.cloudflare.com iburst driftfile /var/lib/chrony/chrony.drift rtcsync diff --git a/ansible/playbooks/update_chrony_client.yaml b/ansible/playbooks/update_chrony_client.yaml index 61301bb..bea60cb 100644 --- a/ansible/playbooks/update_chrony_client.yaml +++ b/ansible/playbooks/update_chrony_client.yaml @@ -1,26 +1,6 @@ --- -- name: Remove conflicting host-level chrony on vaclab-ntp server hosts - hosts: ntp_server_hosts - become: true - tasks: - # These hosts run a vaclab-ntp server pod with hostNetwork: true, whose - # chronyd already disciplines this host's kernel clock directly. A - # second, independent chronyd here would fight it for control of the - # same clock (confirmed: both ended up bound to 127.0.0.1:323 at once). - - name: Stop and disable native chrony service - ansible.builtin.systemd: - name: chrony - state: stopped - enabled: false - ignore_errors: true - - - name: Remove chrony package - ansible.builtin.apt: - name: chrony - state: absent - -- name: Configure chrony NTP client on cluster hosts without a local ntp-server pod - hosts: k3s_cluster:!ntp_server_hosts +- name: Configure chrony NTP client on all cluster hosts + hosts: k3s_cluster become: true handlers: - name: Restart chrony @@ -36,11 +16,10 @@ hosts: k3s_cluster become: true tasks: - - name: Check chrony sources (or note that this host has no host-level chrony) + - name: Check chrony sources ansible.builtin.command: chronyc sources register: chrony_sources changed_when: false - failed_when: false - name: Display chrony sources ansible.builtin.debug: diff --git a/fleet/vaclab-ntp/config/config.yaml b/fleet/vaclab-ntp/config/config.yaml index 1520a05..333a38c 100644 --- a/fleet/vaclab-ntp/config/config.yaml +++ b/fleet/vaclab-ntp/config/config.yaml @@ -4,8 +4,8 @@ metadata: name: vaclab-ntp-server-conf data: chrony.conf: | - server consolidare-continuum.com iburst prefer server time.cloudflare.com iburst + server consolidare-continuum.com iburst driftfile /var/lib/chrony/drift rtcsync makestep 1.0 3 diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index a9d5478..52dcf53 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -13,36 +13,25 @@ spec: labels: app: ntp-server spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - metal-01.he-eu-hel1.misc.vacdst + # No hostNetwork / SYS_TIME / node pinning: this pod never disciplines + # any host's real kernel clock directly (that was the suspected cause + # of the unresolved, still-growing frequency error on metal-01). It's + # a pure NTP relay - syncs from upstream as a client, serves that time + # over the wire via the ntp-server-nodeport Service. Every cluster + # host, including metal-01, syncs against it as a uniform NTP client. containers: - name: ntp-server resources: - # requests.cpu != limits.cpu on purpose: this must NOT be - # Guaranteed QoS. A Guaranteed pod gets an exclusive, pinned - # physical core under the cluster's static CPU manager policy, - # which is suspected of causing this pod's chronyd to measure a - # large, still-growing clock frequency error (~500ppm) despite - # otherwise clean, low-noise measurements. requests: - memory: "512Mi" - cpu: "250m" + memory: "256Mi" + cpu: "100m" limits: - memory: "1Gi" - cpu: "1" + memory: "512Mi" + cpu: "500m" image: cturra/ntp:latest securityContext: capabilities: - add: ["SYS_TIME", "NET_BIND_SERVICE"] + add: ["NET_BIND_SERVICE"] volumeMounts: - name: conf mountPath: /etc/chrony/chrony.conf @@ -62,4 +51,4 @@ spec: storageClassName: longhorn-vaclab resources: requests: - storage: 1Gi \ No newline at end of file + storage: 1Gi diff --git a/fleet/vaclab-ntp/server/service-nodeport.yaml b/fleet/vaclab-ntp/server/service-nodeport.yaml new file mode 100644 index 0000000..6dcfc32 --- /dev/null +++ b/fleet/vaclab-ntp/server/service-nodeport.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: ntp-server-nodeport +spec: + type: NodePort + selector: + app: ntp-server + ports: + - name: ntp + protocol: UDP + port: 123 + targetPort: 123 + nodePort: 30123 From 56f06024c73e907339caa5dfac18ca510054446d Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Tue, 11 Aug 2026 08:11:16 +0000 Subject: [PATCH 15/24] drop hostNetwork for ntp server --- fleet/vaclab-ntp/server/ntp-server.yaml | 40 ++++++++++++++++++------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index 52dcf53..eb7ac4c 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -13,21 +13,41 @@ spec: labels: app: ntp-server spec: - # No hostNetwork / SYS_TIME / node pinning: this pod never disciplines - # any host's real kernel clock directly (that was the suspected cause - # of the unresolved, still-growing frequency error on metal-01). It's - # a pure NTP relay - syncs from upstream as a client, serves that time - # over the wire via the ntp-server-nodeport Service. Every cluster - # host, including metal-01, syncs against it as a uniform NTP client. + # No hostNetwork / SYS_TIME: this pod never disciplines any host's + # real kernel clock directly (that was the suspected cause of the + # unresolved, still-growing frequency error under the old design). + # It's a pure NTP relay - syncs from upstream as a client, serves that + # time over the wire via the ntp-server-nodeport Service. Every + # cluster host, including metal-01, syncs against it as a uniform NTP + # client. + # + # Pinned to metal-01: every client's config points at 65.21.76.50 + # (metal-01's address) regardless of where the pod runs. If the pod + # lands elsewhere, requests take a genuine extra cross-node overlay + # hop (client -> metal-01 -> DNAT -> tunnel -> pod's actual node) for + # every host except whichever one happens to be co-located with the + # pod, which then measures a structurally different (shorter/ + # asymmetric) path than everyone else. Pinning to metal-01 makes that + # hop uniform for the other 5 hosts, so only metal-01 itself is the + # (single, expected) local-delivery outlier. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst containers: - name: ntp-server resources: requests: - memory: "256Mi" - cpu: "100m" + memory: "1Gi" + cpu: "2" limits: - memory: "512Mi" - cpu: "500m" + memory: "1Gi" + cpu: "2" image: cturra/ntp:latest securityContext: capabilities: From 02844af4d89c3b23a1f0bbec2d36aecf1f59ef36 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Tue, 11 Aug 2026 09:00:35 +0000 Subject: [PATCH 16/24] pin ntp on node 01 --- ansible/playbooks/tasks/chrony_client.yaml | 11 +++++++---- fleet/vaclab-ntp/config/config.yaml | 2 +- fleet/vaclab-ntp/server/ntp-server.yaml | 16 ++++++---------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/ansible/playbooks/tasks/chrony_client.yaml b/ansible/playbooks/tasks/chrony_client.yaml index 6c832fb..84420fc 100644 --- a/ansible/playbooks/tasks/chrony_client.yaml +++ b/ansible/playbooks/tasks/chrony_client.yaml @@ -12,10 +12,13 @@ content: | # Managed by Ansible: update_chrony_client.yaml # Primary: vaclab-ntp server pod, reached via its NodePort (30123/udp) - # on metal-01.he-eu-hel1.misc.vacdst. NodePort routes correctly to the - # pod regardless of which node it's actually scheduled on. IP (not - # hostname) to avoid depending on DNS for time sync. - server 65.21.76.50 port 30123 iburst prefer + # on node-01.ih-eu-mda1.misc.vaclab. NodePort routes correctly to the + # pod regardless of which node it's actually scheduled on. Using the + # full SSH-facing hostname (confirmed resolvable - it's what every + # ansible connection in this playbook uses), not the bare k8s node + # name, which is a different string and not confirmed resolvable at + # the OS/DNS level. + server node-01.ih-eu-mda1.misc.vaclab.status.im port 30123 iburst prefer # Backup: only used if the vaclab-ntp server is unreachable. server time.cloudflare.com iburst driftfile /var/lib/chrony/chrony.drift diff --git a/fleet/vaclab-ntp/config/config.yaml b/fleet/vaclab-ntp/config/config.yaml index 333a38c..473a6ad 100644 --- a/fleet/vaclab-ntp/config/config.yaml +++ b/fleet/vaclab-ntp/config/config.yaml @@ -4,8 +4,8 @@ metadata: name: vaclab-ntp-server-conf data: chrony.conf: | - server time.cloudflare.com iburst server consolidare-continuum.com iburst + server time.cloudflare.com iburst driftfile /var/lib/chrony/drift rtcsync makestep 1.0 3 diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index eb7ac4c..483439d 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -21,15 +21,11 @@ spec: # cluster host, including metal-01, syncs against it as a uniform NTP # client. # - # Pinned to metal-01: every client's config points at 65.21.76.50 - # (metal-01's address) regardless of where the pod runs. If the pod - # lands elsewhere, requests take a genuine extra cross-node overlay - # hop (client -> metal-01 -> DNAT -> tunnel -> pod's actual node) for - # every host except whichever one happens to be co-located with the - # pod, which then measures a structurally different (shorter/ - # asymmetric) path than everyone else. Pinning to metal-01 makes that - # hop uniform for the other 5 hosts, so only metal-01 itself is the - # (single, expected) local-delivery outlier. + # Pinned to node-01: node-01..05 are all in the same location + # (ih-eu-mda1); metal-01 is the one outlier, in a different location + # (he-eu-hel1) with a confirmed real ~53ms RTT to the others. Pinning + # the server here gives 4 of 5 client hosts a genuine same-LAN path, + # and only metal-01 carries the long WAN hop, instead of the reverse. affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -38,7 +34,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - metal-01.he-eu-hel1.misc.vacdst + - node-01.ih-eu-mda1.misc.vaclab containers: - name: ntp-server resources: From 87cdf33b6373af36fd43ce2fbd1a8cdb86ea3240 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Tue, 11 Aug 2026 09:14:43 +0000 Subject: [PATCH 17/24] pin ntp on metal 01 --- ansible/inventory.yaml | 8 ++++ ansible/playbooks/chrony_report.yaml | 48 +++++++++++++++++-- ansible/playbooks/tasks/chrony_client.yaml | 15 +++--- ansible/playbooks/update_chrony_client.yaml | 27 +++++++++-- fleet/vaclab-ntp/server/ntp-server.yaml | 34 +++++++------ fleet/vaclab-ntp/server/service-nodeport.yaml | 14 ------ 6 files changed, 104 insertions(+), 42 deletions(-) delete mode 100644 fleet/vaclab-ntp/server/service-nodeport.yaml diff --git a/ansible/inventory.yaml b/ansible/inventory.yaml index 1dfa0c8..0e55a4b 100644 --- a/ansible/inventory.yaml +++ b/ansible/inventory.yaml @@ -1,4 +1,12 @@ --- +# Hosts that run a vaclab-ntp server pod (hostNetwork) locally. Their system +# clock is already disciplined directly by that pod's chronyd, so they must +# NOT also run a native host-level chrony client - two chronyd instances +# steering the same kernel clock fight each other and cause large offsets. +ntp_server_hosts: + hosts: + metal-01.he-eu-hel1.misc.vacdst.status.im: + k3s_cluster: children: server: diff --git a/ansible/playbooks/chrony_report.yaml b/ansible/playbooks/chrony_report.yaml index aaf63e5..972093e 100644 --- a/ansible/playbooks/chrony_report.yaml +++ b/ansible/playbooks/chrony_report.yaml @@ -1,10 +1,10 @@ --- -- name: Gather chrony sync status from all cluster nodes - hosts: k3s_cluster +- name: Gather chrony tracking status from client hosts + hosts: k3s_cluster:!ntp_server_hosts become: true gather_facts: false tasks: - - name: Get chrony tracking summary + - name: Get chrony tracking summary (host-level) ansible.builtin.shell: | set -o pipefail chronyc tracking | awk -F': ' ' @@ -31,6 +31,43 @@ register: chrony_summary changed_when: false +# ntp_server_hosts (currently just metal-01) have no host-level chronyc +# anymore - only the co-located ntp-server pod (replicas: 1, ordinal 0) +# disciplines that host's clock. Query the pod itself instead. +- name: Gather chrony tracking status from the vaclab-ntp server pod + hosts: server[0] + become: false + gather_facts: false + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + tasks: + - name: Get chrony tracking summary from ntp-server-0 + ansible.builtin.shell: | + set -o pipefail + kubectl exec -n vaclab-ntp ntp-server-0 -- chronyc tracking | awk -F': ' ' + /^Stratum/ {stratum=$2} + /^System time/ { + split($2, a, " "); v = a[1] + 0 + if ($2 ~ /fast/) v = -v + sys_ms = v * 1000 + } + /^Last offset/ { + split($2, a, " "); lastoff_ms = (a[1] + 0) * 1000 + } + /^RMS offset/ { + split($2, a, " "); rms_ms = (a[1] + 0) * 1000 + } + /^Root dispersion/ { + split($2, a, " "); rootdisp_ms = (a[1] + 0) * 1000 + } + /^Leap status/ {leap=$2} + END {printf "%s,%.3f,%.3f,%.3f,%.3f,%s", stratum, sys_ms, lastoff_ms, rms_ms, rootdisp_ms, leap} + ' + args: + executable: /bin/bash + register: chrony_summary_ntp_server_pod + changed_when: false + - name: Display cluster-wide chrony sync table hosts: server[0] become: false @@ -43,6 +80,11 @@ msg: | {{ header }} {% for item in groups['k3s_cluster'] %} + {% if item in groups['ntp_server_hosts'] %} + {% set f = chrony_summary_ntp_server_pod.stdout.split(',') %} + {{ '%-32s %-8s %12s %14s %10s %14s %-8s' | format(item | regex_replace('\.status\.im$', '') ~ ' (via pod)', f[0], f[1], f[2], f[3], f[4], f[5]) }} + {% else %} {% set f = hostvars[item].chrony_summary.stdout.split(',') %} {{ '%-32s %-8s %12s %14s %10s %14s %-8s' | format(item | regex_replace('\.status\.im$', ''), f[0], f[1], f[2], f[3], f[4], f[5]) }} + {% endif %} {% endfor %} diff --git a/ansible/playbooks/tasks/chrony_client.yaml b/ansible/playbooks/tasks/chrony_client.yaml index 84420fc..f31dc8e 100644 --- a/ansible/playbooks/tasks/chrony_client.yaml +++ b/ansible/playbooks/tasks/chrony_client.yaml @@ -11,14 +11,13 @@ mode: '0644' content: | # Managed by Ansible: update_chrony_client.yaml - # Primary: vaclab-ntp server pod, reached via its NodePort (30123/udp) - # on node-01.ih-eu-mda1.misc.vaclab. NodePort routes correctly to the - # pod regardless of which node it's actually scheduled on. Using the - # full SSH-facing hostname (confirmed resolvable - it's what every - # ansible connection in this playbook uses), not the bare k8s node - # name, which is a different string and not confirmed resolvable at - # the OS/DNS level. - server node-01.ih-eu-mda1.misc.vaclab.status.im port 30123 iburst prefer + # Primary: the sole vaclab-ntp server, hostNetwork on metal-01, so + # reached directly on the standard NTP port (123) - no NodePort + # involved. Using the full SSH-facing hostname (confirmed resolvable - + # it's what every ansible connection in this playbook uses), not the + # bare k8s node name, which is a different string and not confirmed + # resolvable at the OS/DNS level. + server metal-01.he-eu-hel1.misc.vacdst.status.im iburst prefer # Backup: only used if the vaclab-ntp server is unreachable. server time.cloudflare.com iburst driftfile /var/lib/chrony/chrony.drift diff --git a/ansible/playbooks/update_chrony_client.yaml b/ansible/playbooks/update_chrony_client.yaml index bea60cb..61301bb 100644 --- a/ansible/playbooks/update_chrony_client.yaml +++ b/ansible/playbooks/update_chrony_client.yaml @@ -1,6 +1,26 @@ --- -- name: Configure chrony NTP client on all cluster hosts - hosts: k3s_cluster +- name: Remove conflicting host-level chrony on vaclab-ntp server hosts + hosts: ntp_server_hosts + become: true + tasks: + # These hosts run a vaclab-ntp server pod with hostNetwork: true, whose + # chronyd already disciplines this host's kernel clock directly. A + # second, independent chronyd here would fight it for control of the + # same clock (confirmed: both ended up bound to 127.0.0.1:323 at once). + - name: Stop and disable native chrony service + ansible.builtin.systemd: + name: chrony + state: stopped + enabled: false + ignore_errors: true + + - name: Remove chrony package + ansible.builtin.apt: + name: chrony + state: absent + +- name: Configure chrony NTP client on cluster hosts without a local ntp-server pod + hosts: k3s_cluster:!ntp_server_hosts become: true handlers: - name: Restart chrony @@ -16,10 +36,11 @@ hosts: k3s_cluster become: true tasks: - - name: Check chrony sources + - name: Check chrony sources (or note that this host has no host-level chrony) ansible.builtin.command: chronyc sources register: chrony_sources changed_when: false + failed_when: false - name: Display chrony sources ansible.builtin.debug: diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index 483439d..ea33352 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -13,19 +13,25 @@ spec: labels: app: ntp-server spec: - # No hostNetwork / SYS_TIME: this pod never disciplines any host's - # real kernel clock directly (that was the suspected cause of the - # unresolved, still-growing frequency error under the old design). - # It's a pure NTP relay - syncs from upstream as a client, serves that - # time over the wire via the ntp-server-nodeport Service. Every - # cluster host, including metal-01, syncs against it as a uniform NTP - # client. + # hostNetwork + SYS_TIME: this pod directly disciplines metal-01's + # real kernel clock. Without SYS_TIME, chronyd can only *measure* + # drift and mathematically compensate for it in served responses, but + # never actually correct the underlying clock. Confirmed by direct + # testing on TWO different nodes (metal-01, then node-01 after + # relocating to rule out WAN-latency as the cause): both showed the + # same unbounded, worsening divergence (skew climbing past 100-800ppm, + # root dispersion into the tens-to-hundreds of ms) instead of + # converging. A proven-working makestep test earlier showed + # SYS_TIME/adjtimex genuinely works when present (landed a real + # 1.169s -> 2ms correction), so this reverts to that design. # - # Pinned to node-01: node-01..05 are all in the same location - # (ih-eu-mda1); metal-01 is the one outlier, in a different location - # (he-eu-hel1) with a confirmed real ~53ms RTT to the others. Pinning - # the server here gives 4 of 5 client hosts a genuine same-LAN path, - # and only metal-01 carries the long WAN hop, instead of the reverse. + # Because this pod owns metal-01's real port 123 via hostNetwork, + # metal-01 must NOT also run a native host-level chrony client - see + # ansible/inventory.yaml's ntp_server_hosts group. Two daemons + # disciplining the same kernel clock fight each other (confirmed via + # ss -ulnp showing two chronyd PIDs both bound to 127.0.0.1:323). + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -34,7 +40,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - node-01.ih-eu-mda1.misc.vaclab + - metal-01.he-eu-hel1.misc.vacdst containers: - name: ntp-server resources: @@ -47,7 +53,7 @@ spec: image: cturra/ntp:latest securityContext: capabilities: - add: ["NET_BIND_SERVICE"] + add: ["SYS_TIME", "NET_BIND_SERVICE"] volumeMounts: - name: conf mountPath: /etc/chrony/chrony.conf diff --git a/fleet/vaclab-ntp/server/service-nodeport.yaml b/fleet/vaclab-ntp/server/service-nodeport.yaml deleted file mode 100644 index 6dcfc32..0000000 --- a/fleet/vaclab-ntp/server/service-nodeport.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: ntp-server-nodeport -spec: - type: NodePort - selector: - app: ntp-server - ports: - - name: ntp - protocol: UDP - port: 123 - targetPort: 123 - nodePort: 30123 From a689535653b4fcde69c8c2361b1ee19e617e6e8e Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Tue, 11 Aug 2026 10:25:12 +0000 Subject: [PATCH 18/24] make ntp server unconfined --- fleet/vaclab-ntp/server/ntp-server.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index ea33352..82814e9 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -54,6 +54,18 @@ spec: securityContext: capabilities: add: ["SYS_TIME", "NET_BIND_SERVICE"] + # SYS_TIME alone isn't enough: since glibc 2.31, adjtimex() calls + # the clock_adjtime syscall, which containerd's default seccomp + # profile doesn't whitelist even under CAP_SYS_TIME (moby/moby + # #40919, #40929). This let discrete step corrections through + # (clock_settime, a different syscall - confirmed working via + # the earlier makestep test) while silently blocking the + # continuous ADJ_FREQUENCY corrections chronyd needs to keep + # drift bounded between polls, matching exactly what we + # observed: confident, unchanging Frequency estimate, tiny + # Last offset each poll, but System time growing unboundedly. + seccompProfile: + type: Unconfined volumeMounts: - name: conf mountPath: /etc/chrony/chrony.conf From 61a8611c275afd577c0f3dba3a5eee81655fd017 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Tue, 11 Aug 2026 10:37:27 +0000 Subject: [PATCH 19/24] make ntp server unconfined --- fleet/vaclab-ntp/server/ntp-server.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index 82814e9..3a08f9a 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -51,6 +51,26 @@ spec: memory: "1Gi" cpu: "2" image: cturra/ntp:latest + # The image's own /opt/startup.sh unconditionally execs + # `chronyd -u chrony -d -x -L ...` - the -x flag is hardcoded, not + # controlled by any env var, and disables clock control + # regardless of SYS_TIME/seccomp granted at the pod level. This + # replicates the script's permission-fixup steps (needed for the + # ntp-drift PVC and /run/chrony) but drops -x. Our own ConfigMap + # already provides the full chrony.conf, so the script's env-var + # -driven config generation isn't needed either. + command: ["/bin/sh", "-c"] + args: + - | + if [ -d /run/chrony ]; then + chown -R chrony:chrony /run/chrony + chmod o-rx /run/chrony + rm -f /var/run/chrony/chronyd.pid + fi + if [ -d /var/lib/chrony ]; then + chown -R chrony:chrony /var/lib/chrony + fi + exec /usr/sbin/chronyd -u chrony -d -L 0 securityContext: capabilities: add: ["SYS_TIME", "NET_BIND_SERVICE"] From 414353d6acb1d45e27ac7cf47cf7ba0a8f100a08 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Sat, 15 Aug 2026 19:28:39 +0000 Subject: [PATCH 20/24] fix otlp collector --- extras/otlp-trace-test-job.yaml | 2 +- fleet/otlp-collector-external/fleet.yaml | 16 ++++++++++++++++ fleet/otlp-collector/fleet.yaml | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/extras/otlp-trace-test-job.yaml b/extras/otlp-trace-test-job.yaml index 99c08fa..d0fb5c8 100644 --- a/extras/otlp-trace-test-job.yaml +++ b/extras/otlp-trace-test-job.yaml @@ -23,7 +23,7 @@ spec: key: token args: - "traces" - - "--otlp-endpoint=otlp.lab.vac.dev:30317" + - "--otlp-endpoint=otlp.lab.vac.dev:31317" - "--otlp-insecure" - "--otlp-header" - "Authorization=\"Bearer $(OTEL_CLIENT_TOKEN)\"" diff --git a/fleet/otlp-collector-external/fleet.yaml b/fleet/otlp-collector-external/fleet.yaml index f219e6e..41bfe75 100644 --- a/fleet/otlp-collector-external/fleet.yaml +++ b/fleet/otlp-collector-external/fleet.yaml @@ -25,6 +25,22 @@ helm: logsCollection: enabled: false + # Same chart as fleet/otlp-collector: default probe timing + # (timeoutSeconds: 1, no initialDelaySeconds) proved too tight for this + # collector on a busy cluster and caused a crash-restart loop the first + # time it was restarted after a long time running. Applying the same + # fix preventively here. + livenessProbe: + initialDelaySeconds: 20 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + readinessProbe: + initialDelaySeconds: 20 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + service: type: ClusterIP annotations: diff --git a/fleet/otlp-collector/fleet.yaml b/fleet/otlp-collector/fleet.yaml index 25299c0..07acaf9 100644 --- a/fleet/otlp-collector/fleet.yaml +++ b/fleet/otlp-collector/fleet.yaml @@ -21,6 +21,25 @@ helm: logsCollection: enabled: true + # Chart defaults (timeoutSeconds: 1, no initialDelaySeconds) are too + # tight for this collector: it discovers/attaches filelog receivers for + # every pod's logs across the whole cluster at startup, which can + # briefly block the health_check HTTP handler past a 1s timeout on a + # busy cluster - causing a crash-restart loop (confirmed: exit code 2 at + # exactly 30s = failureThreshold 3 x periodSeconds 10, clean startup + # logs otherwise, negligible CPU/memory usage - not an actual crash or + # resource issue, just an over-eager probe). + livenessProbe: + initialDelaySeconds: 20 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + readinessProbe: + initialDelaySeconds: 20 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + service: type: ClusterIP From 58213c89da43678c75b3390d2c118fa5c45de777 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Sun, 16 Aug 2026 18:37:58 +0000 Subject: [PATCH 21/24] update RLN generation time panels and add RLN Verification panels --- .../public-internal-mix-testnet.yaml | 636 +++++++++++++----- 1 file changed, 483 insertions(+), 153 deletions(-) diff --git a/fleet/grafana-dashboards/public-internal-mix-testnet.yaml b/fleet/grafana-dashboards/public-internal-mix-testnet.yaml index 50f20c9..8551ade 100644 --- a/fleet/grafana-dashboards/public-internal-mix-testnet.yaml +++ b/fleet/grafana-dashboards/public-internal-mix-testnet.yaml @@ -720,7 +720,7 @@ data: "h": 8, "w": 14, "x": 0, - "y": 108 + "y": 26 }, "id": 10, "options": { @@ -788,7 +788,7 @@ data: "h": 5, "w": 10, "x": 14, - "y": 108 + "y": 26 }, "id": 53, "options": { @@ -856,7 +856,7 @@ data: "h": 3, "w": 10, "x": 14, - "y": 121 + "y": 31 }, "id": 54, "options": { @@ -946,7 +946,7 @@ data: "h": 4, "w": 24, "x": 0, - "y": 169 + "y": 203 }, "id": 52, "options": { @@ -1095,7 +1095,7 @@ data: "h": 6, "w": 24, "x": 0, - "y": 173 + "y": 207 }, "id": 9, "options": { @@ -1247,7 +1247,7 @@ data: "h": 6, "w": 12, "x": 0, - "y": 170 + "y": 204 }, "id": 22, "options": { @@ -1346,7 +1346,7 @@ data: "h": 6, "w": 12, "x": 12, - "y": 170 + "y": 204 }, "id": 21, "options": { @@ -1641,15 +1641,135 @@ data: }, "id": 61, "panels": [], - "title": "RLN Proof", + "title": "RLN Proof Generation", "type": "row" }, { "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "VictoriaLogs" + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "description": "per node histogram can be obtained by setting pod selector to a target node instead of All", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ + { + "id": "displayName", + "value": "${__field.labels.pod}" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(mix_rln_proof_generation_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le,mode))", + "instant": false, + "legendFormat": "p50 ({{mode}}) ", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(mix_rln_proof_generation_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le,mode))", + "hide": false, + "instant": false, + "legendFormat": "p95 ({{mode}}) ", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(mix_rln_proof_generation_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le, mode))", + "hide": false, + "instant": false, + "legendFormat": "p99 ({{mode}}) ", + "range": true, + "refId": "C" + } + ], + "title": "Proof Generation Duration Histogram", + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" }, - "description": "Max is 1 hour. Fetching more than last 1 hour is not supported for now.", + "description": "SUM/COUNT", "fieldConfig": { "defaults": { "color": { @@ -1664,7 +1784,7 @@ data: "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 4, + "fillOpacity": 5, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1703,7 +1823,7 @@ data: } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [ { @@ -1721,18 +1841,18 @@ data: ] }, "gridPos": { - "h": 7, + "h": 6, "w": 17, "x": 0, - "y": 38 + "y": 44 }, - "id": 64, + "id": 66, "options": { "legend": { "calcs": [ "min", - "max", - "mean" + "mean", + "max" ], "displayMode": "table", "placement": "bottom", @@ -1748,90 +1868,26 @@ data: "targets": [ { "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "VictoriaLogs" + "type": "prometheus", + "uid": "VictoriaMetrics" }, "editorMode": "code", - "expr": "kubernetes.pod_namespace:\"$namespace\" AND \"RLN proof\"\n| copy _time as start_marker\n| union (\n kubernetes.pod_namespace:\"$namespace\" AND \"proof generated successfully\"\n )\n| replace (\"\\\"\", \"\") at kubernetes.pod_name\n| sort by (kubernetes.pod_name, _time)\n| running_stats by (kubernetes.pod_name) max(start_marker) as latest_start\n| filter \"proof generated successfully\"\n| math (_time - latest_start) / 1000000 as duration_ms\n| sort by (_time) desc\n", - "maxLines": 10000, - "queryType": "instant", + "expr": "sum(rate(mix_rln_proof_generation_duration_seconds_sum{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (instance)\n/\nsum(rate(mix_rln_proof_generation_duration_seconds_count{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (instance)", + "instant": false, + "legendFormat": "__auto", + "range": true, "refId": "A" } ], - "title": "Proof Generation Time", - "transformations": [ - { - "id": "extractFields", - "options": { - "source": "labels" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": { - "Time": true, - "duration_ms": true, - "kubernetes.pod_name": true, - "kubernetes.pod_namespace": true - }, - "indexByName": { - "Time": 0, - "duration_ms": 1, - "kubernetes.pod_name": 3, - "kubernetes.pod_namespace": 2 - }, - "renameByName": { - "duration_ms": "", - "kubernetes.pod_name": "pod", - "kubernetes.pod_namespace": "namespace" - } - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time" - }, - { - "destinationType": "number", - "targetField": "duration_ms" - }, - { - "destinationType": "string", - "targetField": "namespace" - }, - { - "destinationType": "string", - "targetField": "pod" - } - ], - "fields": {} - } - }, - { - "id": "partitionByValues", - "options": { - "fields": [ - "pod", - "namespace" - ], - "keepFields": false - } - } - ], + "title": "Avgerage Proof Generation Duration (Over Time)", "type": "timeseries" }, { "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "VictoriaLogs" + "type": "prometheus", + "uid": "VictoriaMetrics" }, - "description": "Max is 1 hour. Fetching more than last 1 hour is not supported for now.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1851,7 +1907,7 @@ data: } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [ { @@ -1869,14 +1925,14 @@ data: ] }, "gridPos": { - "h": 7, + "h": 6, "w": 7, "x": 17, - "y": 38 + "y": 44 }, - "id": 65, + "id": 67, "options": { - "displayMode": "basic", + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", @@ -1889,9 +1945,7 @@ data: "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "p95" - ], + "calcs": [], "fields": "", "values": false }, @@ -1903,82 +1957,358 @@ data: "targets": [ { "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "VictoriaLogs" + "type": "prometheus", + "uid": "VictoriaMetrics" }, "editorMode": "code", - "expr": "kubernetes.pod_namespace:\"$namespace\" AND \"RLN proof\"\n| copy _time as start_marker\n| union (\n kubernetes.pod_namespace:\"$namespace\" AND \"proof generated successfully\"\n )\n| replace (\"\\\"\", \"\") at kubernetes.pod_name\n| sort by (kubernetes.pod_name, _time)\n| running_stats by (kubernetes.pod_name) max(start_marker) as latest_start\n| filter \"proof generated successfully\"\n| math (_time - latest_start) / 1000000 as duration_ms\n| sort by (_time) desc\n", - "maxLines": 10000, - "queryType": "instant", + "expr": "histogram_quantile(0.95,\n sum(rate(mix_rln_proof_generation_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le, instance)\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, "refId": "A" } ], - "title": "Proof Generation Time (95th Percentile)", - "transformations": [ - { - "id": "extractFields", - "options": { - "source": "labels" + "title": "Proof Generation Duration p95", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 68, + "panels": [], + "title": "RLN Proof Verification", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "description": "per node histogram can be obtained by setting pod selector to a target node instead of All", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ + { + "id": "displayName", + "value": "${__field.labels.pod}" + } + ] } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(mix_rln_proof_verification_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le))", + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" }, { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": { - "Time": true, - "duration_ms": true, - "kubernetes.pod_name": true, - "kubernetes.pod_namespace": true + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(mix_rln_proof_verification_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(mix_rln_proof_verification_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Proof Verification Duration Histogram", + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "description": "SUM/COUNT", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "indexByName": { - "Time": 0, - "duration_ms": 1, - "kubernetes.pod_name": 3, - "kubernetes.pod_namespace": 2 + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "renameByName": { - "duration_ms": "", - "kubernetes.pod_name": "pod", - "kubernetes.pod_namespace": "namespace" + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ + { + "id": "displayName", + "value": "${__field.labels.pod}" + } + ] } + ] + }, + "gridPos": { + "h": 6, + "w": 17, + "x": 0, + "y": 57 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time" - }, + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "sum(rate(mix_rln_proof_verification_duration_seconds_sum{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (instance)\n/\nsum(rate(mix_rln_proof_verification_duration_seconds_count{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (instance)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Avgerage Proof Verification Duration (Over Time)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "destinationType": "number", - "targetField": "duration_ms" + "color": "green", + "value": 0 }, { - "destinationType": "string", - "targetField": "namespace" - }, + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ { - "destinationType": "string", - "targetField": "pod" + "id": "displayName", + "value": "${__field.labels.pod}" } - ], - "fields": {} + ] } + ] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 17, + "y": 57 + }, + "id": 71, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.3.0", + "targets": [ { - "id": "partitionByValues", - "options": { - "fields": [ - "pod", - "namespace" - ], - "keepFields": false - } + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95,\n sum(rate(mix_rln_proof_verification_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le, instance)\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" } ], + "title": "Proof Verification Duration p95", "type": "bargauge" } ], @@ -2036,5 +2366,5 @@ data: "timezone": "browser", "title": "Internal - MIX testnet 0.2", "uid": "akgtzjg29062026", - "version": 11 + "version": 12 } \ No newline at end of file From 49c3819cfe0db1caea9eac7587ea3fb4f76a9170 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Sun, 16 Aug 2026 19:11:58 +0000 Subject: [PATCH 22/24] update otlp collectors comments --- fleet/otlp-collector-external/fleet.yaml | 4 ---- fleet/otlp-collector/fleet.yaml | 8 +------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/fleet/otlp-collector-external/fleet.yaml b/fleet/otlp-collector-external/fleet.yaml index 41bfe75..7efac70 100644 --- a/fleet/otlp-collector-external/fleet.yaml +++ b/fleet/otlp-collector-external/fleet.yaml @@ -25,11 +25,7 @@ helm: logsCollection: enabled: false - # Same chart as fleet/otlp-collector: default probe timing # (timeoutSeconds: 1, no initialDelaySeconds) proved too tight for this - # collector on a busy cluster and caused a crash-restart loop the first - # time it was restarted after a long time running. Applying the same - # fix preventively here. livenessProbe: initialDelaySeconds: 20 timeoutSeconds: 5 diff --git a/fleet/otlp-collector/fleet.yaml b/fleet/otlp-collector/fleet.yaml index 07acaf9..1af2f5b 100644 --- a/fleet/otlp-collector/fleet.yaml +++ b/fleet/otlp-collector/fleet.yaml @@ -22,13 +22,7 @@ helm: enabled: true # Chart defaults (timeoutSeconds: 1, no initialDelaySeconds) are too - # tight for this collector: it discovers/attaches filelog receivers for - # every pod's logs across the whole cluster at startup, which can - # briefly block the health_check HTTP handler past a 1s timeout on a - # busy cluster - causing a crash-restart loop (confirmed: exit code 2 at - # exactly 30s = failureThreshold 3 x periodSeconds 10, clean startup - # logs otherwise, negligible CPU/memory usage - not an actual crash or - # resource issue, just an over-eager probe). + # tight for this collector livenessProbe: initialDelaySeconds: 20 timeoutSeconds: 5 From 6a5f1fda2794bec9b49079568e0e7970eb84c7a6 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Sun, 16 Aug 2026 19:40:34 +0000 Subject: [PATCH 23/24] update ntp comments --- ansible/playbooks/update_chrony_client.yaml | 7 ++-- fleet/vaclab-ntp/server/ntp-server.yaml | 37 ++++----------------- 2 files changed, 10 insertions(+), 34 deletions(-) diff --git a/ansible/playbooks/update_chrony_client.yaml b/ansible/playbooks/update_chrony_client.yaml index 61301bb..7b9dad0 100644 --- a/ansible/playbooks/update_chrony_client.yaml +++ b/ansible/playbooks/update_chrony_client.yaml @@ -4,9 +4,10 @@ become: true tasks: # These hosts run a vaclab-ntp server pod with hostNetwork: true, whose - # chronyd already disciplines this host's kernel clock directly. A - # second, independent chronyd here would fight it for control of the - # same clock (confirmed: both ended up bound to 127.0.0.1:323 at once). + # chronyd already corrects this host's kernel clock directly. A + # second, independent chronyd here would fight it for control over the + # same clock and cause instability. + # Disabling the host-level chrony client avoids this conflict. - name: Stop and disable native chrony service ansible.builtin.systemd: name: chrony diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index 3a08f9a..a18cadc 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -13,23 +13,15 @@ spec: labels: app: ntp-server spec: - # hostNetwork + SYS_TIME: this pod directly disciplines metal-01's + # hostNetwork + SYS_TIME: this pod directly corrects metal-01's # real kernel clock. Without SYS_TIME, chronyd can only *measure* # drift and mathematically compensate for it in served responses, but - # never actually correct the underlying clock. Confirmed by direct - # testing on TWO different nodes (metal-01, then node-01 after - # relocating to rule out WAN-latency as the cause): both showed the - # same unbounded, worsening divergence (skew climbing past 100-800ppm, - # root dispersion into the tens-to-hundreds of ms) instead of - # converging. A proven-working makestep test earlier showed - # SYS_TIME/adjtimex genuinely works when present (landed a real - # 1.169s -> 2ms correction), so this reverts to that design. - # + # never actually correct the underlying clock. # Because this pod owns metal-01's real port 123 via hostNetwork, # metal-01 must NOT also run a native host-level chrony client - see - # ansible/inventory.yaml's ntp_server_hosts group. Two daemons - # disciplining the same kernel clock fight each other (confirmed via - # ss -ulnp showing two chronyd PIDs both bound to 127.0.0.1:323). + # ansible/inventory.yaml's ntp_server_hosts group. 2 daemons + # disciplining the same kernel clock fight each other and cause instability. + # The host-level chrony client is disabled on metal-01. hostNetwork: true dnsPolicy: ClusterFirstWithHostNet affinity: @@ -51,14 +43,6 @@ spec: memory: "1Gi" cpu: "2" image: cturra/ntp:latest - # The image's own /opt/startup.sh unconditionally execs - # `chronyd -u chrony -d -x -L ...` - the -x flag is hardcoded, not - # controlled by any env var, and disables clock control - # regardless of SYS_TIME/seccomp granted at the pod level. This - # replicates the script's permission-fixup steps (needed for the - # ntp-drift PVC and /run/chrony) but drops -x. Our own ConfigMap - # already provides the full chrony.conf, so the script's env-var - # -driven config generation isn't needed either. command: ["/bin/sh", "-c"] args: - | @@ -74,16 +58,7 @@ spec: securityContext: capabilities: add: ["SYS_TIME", "NET_BIND_SERVICE"] - # SYS_TIME alone isn't enough: since glibc 2.31, adjtimex() calls - # the clock_adjtime syscall, which containerd's default seccomp - # profile doesn't whitelist even under CAP_SYS_TIME (moby/moby - # #40919, #40929). This let discrete step corrections through - # (clock_settime, a different syscall - confirmed working via - # the earlier makestep test) while silently blocking the - # continuous ADJ_FREQUENCY corrections chronyd needs to keep - # drift bounded between polls, matching exactly what we - # observed: confident, unchanging Frequency estimate, tiny - # Last offset each poll, but System time growing unboundedly. + # SYS_TIME alone isn't enough seccompProfile: type: Unconfined volumeMounts: From 41b049635dfb6f1aac49b444ec89157da0b9d163 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Sun, 16 Aug 2026 19:45:55 +0000 Subject: [PATCH 24/24] deploy dst dashboard backend v1.4 and frontend v0.6 --- fleet/dst-dashboard/backend/backend.yaml | 2 +- fleet/dst-dashboard/frontend/frontend.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fleet/dst-dashboard/backend/backend.yaml b/fleet/dst-dashboard/backend/backend.yaml index 3b34778..badc72e 100644 --- a/fleet/dst-dashboard/backend/backend.yaml +++ b/fleet/dst-dashboard/backend/backend.yaml @@ -30,7 +30,7 @@ spec: effect: "PreferNoSchedule" containers: - name: dst-dashboard-backend - image: mamoutoudiarra/dst-dashboard-backend:v1.3 + image: mamoutoudiarra/dst-dashboard-backend:v1.4 imagePullPolicy: IfNotPresent ports: - containerPort: 8000 diff --git a/fleet/dst-dashboard/frontend/frontend.yaml b/fleet/dst-dashboard/frontend/frontend.yaml index bd402fe..356eae0 100644 --- a/fleet/dst-dashboard/frontend/frontend.yaml +++ b/fleet/dst-dashboard/frontend/frontend.yaml @@ -30,7 +30,7 @@ spec: effect: "PreferNoSchedule" containers: - name: frontend - image: mamoutoudiarra/dst-dashboard-frontend:v0.5 + image: mamoutoudiarra/dst-dashboard-frontend:v0.6 imagePullPolicy: IfNotPresent ports: - containerPort: 80