diff --git a/ansible/inventory.yaml b/ansible/inventory.yaml index 1dfa0c8..0e55a4b 100644 --- a/ansible/inventory.yaml +++ b/ansible/inventory.yaml @@ -1,4 +1,12 @@ --- +# Hosts that run a vaclab-ntp server pod (hostNetwork) locally. Their system +# clock is already disciplined directly by that pod's chronyd, so they must +# NOT also run a native host-level chrony client - two chronyd instances +# steering the same kernel clock fight each other and cause large offsets. +ntp_server_hosts: + hosts: + metal-01.he-eu-hel1.misc.vacdst.status.im: + k3s_cluster: children: server: diff --git a/ansible/playbooks/chrony_report.yaml b/ansible/playbooks/chrony_report.yaml new file mode 100644 index 0000000..972093e --- /dev/null +++ b/ansible/playbooks/chrony_report.yaml @@ -0,0 +1,90 @@ +--- +- name: Gather chrony tracking status from client hosts + hosts: k3s_cluster:!ntp_server_hosts + become: true + gather_facts: false + tasks: + - name: Get chrony tracking summary (host-level) + ansible.builtin.shell: | + set -o pipefail + chronyc tracking | awk -F': ' ' + /^Stratum/ {stratum=$2} + /^System time/ { + split($2, a, " "); v = a[1] + 0 + if ($2 ~ /fast/) v = -v + sys_ms = v * 1000 + } + /^Last offset/ { + split($2, a, " "); lastoff_ms = (a[1] + 0) * 1000 + } + /^RMS offset/ { + split($2, a, " "); rms_ms = (a[1] + 0) * 1000 + } + /^Root dispersion/ { + split($2, a, " "); rootdisp_ms = (a[1] + 0) * 1000 + } + /^Leap status/ {leap=$2} + END {printf "%s,%.3f,%.3f,%.3f,%.3f,%s", stratum, sys_ms, lastoff_ms, rms_ms, rootdisp_ms, leap} + ' + args: + executable: /bin/bash + register: chrony_summary + changed_when: false + +# ntp_server_hosts (currently just metal-01) have no host-level chronyc +# anymore - only the co-located ntp-server pod (replicas: 1, ordinal 0) +# disciplines that host's clock. Query the pod itself instead. +- name: Gather chrony tracking status from the vaclab-ntp server pod + hosts: server[0] + become: false + gather_facts: false + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + tasks: + - name: Get chrony tracking summary from ntp-server-0 + ansible.builtin.shell: | + set -o pipefail + kubectl exec -n vaclab-ntp ntp-server-0 -- chronyc tracking | awk -F': ' ' + /^Stratum/ {stratum=$2} + /^System time/ { + split($2, a, " "); v = a[1] + 0 + if ($2 ~ /fast/) v = -v + sys_ms = v * 1000 + } + /^Last offset/ { + split($2, a, " "); lastoff_ms = (a[1] + 0) * 1000 + } + /^RMS offset/ { + split($2, a, " "); rms_ms = (a[1] + 0) * 1000 + } + /^Root dispersion/ { + split($2, a, " "); rootdisp_ms = (a[1] + 0) * 1000 + } + /^Leap status/ {leap=$2} + END {printf "%s,%.3f,%.3f,%.3f,%.3f,%s", stratum, sys_ms, lastoff_ms, rms_ms, rootdisp_ms, leap} + ' + args: + executable: /bin/bash + register: chrony_summary_ntp_server_pod + changed_when: false + +- name: Display cluster-wide chrony sync table + hosts: server[0] + become: false + gather_facts: false + tasks: + - name: Print combined table + vars: + header: "{{ '%-32s %-8s %12s %14s %10s %14s %-8s' | format('HOST', 'STRATUM', 'OFFSET(ms)', 'LASTOFF(ms)', 'RMS(ms)', 'ROOTDISP(ms)', 'LEAP') }}" + ansible.builtin.debug: + msg: | + {{ header }} + {% for item in groups['k3s_cluster'] %} + {% if item in groups['ntp_server_hosts'] %} + {% set f = chrony_summary_ntp_server_pod.stdout.split(',') %} + {{ '%-32s %-8s %12s %14s %10s %14s %-8s' | format(item | regex_replace('\.status\.im$', '') ~ ' (via pod)', f[0], f[1], f[2], f[3], f[4], f[5]) }} + {% else %} + {% set f = hostvars[item].chrony_summary.stdout.split(',') %} + {{ '%-32s %-8s %12s %14s %10s %14s %-8s' | format(item | regex_replace('\.status\.im$', ''), f[0], f[1], f[2], f[3], f[4], f[5]) }} + {% endif %} + {% endfor %} diff --git a/ansible/playbooks/tasks/chrony_client.yaml b/ansible/playbooks/tasks/chrony_client.yaml new file mode 100644 index 0000000..f31dc8e --- /dev/null +++ b/ansible/playbooks/tasks/chrony_client.yaml @@ -0,0 +1,33 @@ +--- +- name: Install chrony + ansible.builtin.apt: + name: chrony + state: present + update_cache: true + +- name: Configure chrony to sync against the internal vaclab-ntp server, with Cloudflare as backup + ansible.builtin.copy: + dest: /etc/chrony/chrony.conf + mode: '0644' + content: | + # Managed by Ansible: update_chrony_client.yaml + # Primary: the sole vaclab-ntp server, hostNetwork on metal-01, so + # reached directly on the standard NTP port (123) - no NodePort + # involved. Using the full SSH-facing hostname (confirmed resolvable - + # it's what every ansible connection in this playbook uses), not the + # bare k8s node name, which is a different string and not confirmed + # resolvable at the OS/DNS level. + server metal-01.he-eu-hel1.misc.vacdst.status.im iburst prefer + # Backup: only used if the vaclab-ntp server is unreachable. + server time.cloudflare.com iburst + driftfile /var/lib/chrony/chrony.drift + rtcsync + makestep 1.0 3 + logdir /var/log/chrony + notify: Restart chrony + +- name: Ensure chrony service is enabled and running + ansible.builtin.systemd: + name: chrony + state: started + enabled: true diff --git a/ansible/playbooks/update_chrony_client.yaml b/ansible/playbooks/update_chrony_client.yaml new file mode 100644 index 0000000..7b9dad0 --- /dev/null +++ b/ansible/playbooks/update_chrony_client.yaml @@ -0,0 +1,48 @@ +--- +- name: Remove conflicting host-level chrony on vaclab-ntp server hosts + hosts: ntp_server_hosts + become: true + tasks: + # These hosts run a vaclab-ntp server pod with hostNetwork: true, whose + # chronyd already corrects this host's kernel clock directly. A + # second, independent chronyd here would fight it for control over the + # same clock and cause instability. + # Disabling the host-level chrony client avoids this conflict. + - name: Stop and disable native chrony service + ansible.builtin.systemd: + name: chrony + state: stopped + enabled: false + ignore_errors: true + + - name: Remove chrony package + ansible.builtin.apt: + name: chrony + state: absent + +- name: Configure chrony NTP client on cluster hosts without a local ntp-server pod + hosts: k3s_cluster:!ntp_server_hosts + become: true + handlers: + - name: Restart chrony + ansible.builtin.systemd: + name: chrony + state: restarted + + tasks: + - name: Apply chrony client configuration + ansible.builtin.import_tasks: tasks/chrony_client.yaml + +- name: Verify chrony sync status + hosts: k3s_cluster + become: true + tasks: + - name: Check chrony sources (or note that this host has no host-level chrony) + ansible.builtin.command: chronyc sources + register: chrony_sources + changed_when: false + failed_when: false + + - name: Display chrony sources + ansible.builtin.debug: + var: chrony_sources.stdout_lines diff --git a/extras/otlp-trace-test-job.yaml b/extras/otlp-trace-test-job.yaml index 99c08fa..d0fb5c8 100644 --- a/extras/otlp-trace-test-job.yaml +++ b/extras/otlp-trace-test-job.yaml @@ -23,7 +23,7 @@ spec: key: token args: - "traces" - - "--otlp-endpoint=otlp.lab.vac.dev:30317" + - "--otlp-endpoint=otlp.lab.vac.dev:31317" - "--otlp-insecure" - "--otlp-header" - "Authorization=\"Bearer $(OTEL_CLIENT_TOKEN)\"" diff --git a/fleet/dst-dashboard/backend/backend.yaml b/fleet/dst-dashboard/backend/backend.yaml index 3b34778..badc72e 100644 --- a/fleet/dst-dashboard/backend/backend.yaml +++ b/fleet/dst-dashboard/backend/backend.yaml @@ -30,7 +30,7 @@ spec: effect: "PreferNoSchedule" containers: - name: dst-dashboard-backend - image: mamoutoudiarra/dst-dashboard-backend:v1.3 + image: mamoutoudiarra/dst-dashboard-backend:v1.4 imagePullPolicy: IfNotPresent ports: - containerPort: 8000 diff --git a/fleet/dst-dashboard/frontend/frontend.yaml b/fleet/dst-dashboard/frontend/frontend.yaml index bd402fe..356eae0 100644 --- a/fleet/dst-dashboard/frontend/frontend.yaml +++ b/fleet/dst-dashboard/frontend/frontend.yaml @@ -30,7 +30,7 @@ spec: effect: "PreferNoSchedule" containers: - name: frontend - image: mamoutoudiarra/dst-dashboard-frontend:v0.5 + image: mamoutoudiarra/dst-dashboard-frontend:v0.6 imagePullPolicy: IfNotPresent ports: - containerPort: 80 diff --git a/fleet/grafana-dashboards/public-internal-mix-testnet.yaml b/fleet/grafana-dashboards/public-internal-mix-testnet.yaml index 50f20c9..8551ade 100644 --- a/fleet/grafana-dashboards/public-internal-mix-testnet.yaml +++ b/fleet/grafana-dashboards/public-internal-mix-testnet.yaml @@ -720,7 +720,7 @@ data: "h": 8, "w": 14, "x": 0, - "y": 108 + "y": 26 }, "id": 10, "options": { @@ -788,7 +788,7 @@ data: "h": 5, "w": 10, "x": 14, - "y": 108 + "y": 26 }, "id": 53, "options": { @@ -856,7 +856,7 @@ data: "h": 3, "w": 10, "x": 14, - "y": 121 + "y": 31 }, "id": 54, "options": { @@ -946,7 +946,7 @@ data: "h": 4, "w": 24, "x": 0, - "y": 169 + "y": 203 }, "id": 52, "options": { @@ -1095,7 +1095,7 @@ data: "h": 6, "w": 24, "x": 0, - "y": 173 + "y": 207 }, "id": 9, "options": { @@ -1247,7 +1247,7 @@ data: "h": 6, "w": 12, "x": 0, - "y": 170 + "y": 204 }, "id": 22, "options": { @@ -1346,7 +1346,7 @@ data: "h": 6, "w": 12, "x": 12, - "y": 170 + "y": 204 }, "id": 21, "options": { @@ -1641,15 +1641,135 @@ data: }, "id": 61, "panels": [], - "title": "RLN Proof", + "title": "RLN Proof Generation", "type": "row" }, { "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "VictoriaLogs" + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "description": "per node histogram can be obtained by setting pod selector to a target node instead of All", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ + { + "id": "displayName", + "value": "${__field.labels.pod}" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(mix_rln_proof_generation_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le,mode))", + "instant": false, + "legendFormat": "p50 ({{mode}}) ", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(mix_rln_proof_generation_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le,mode))", + "hide": false, + "instant": false, + "legendFormat": "p95 ({{mode}}) ", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(mix_rln_proof_generation_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le, mode))", + "hide": false, + "instant": false, + "legendFormat": "p99 ({{mode}}) ", + "range": true, + "refId": "C" + } + ], + "title": "Proof Generation Duration Histogram", + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" }, - "description": "Max is 1 hour. Fetching more than last 1 hour is not supported for now.", + "description": "SUM/COUNT", "fieldConfig": { "defaults": { "color": { @@ -1664,7 +1784,7 @@ data: "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 4, + "fillOpacity": 5, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1703,7 +1823,7 @@ data: } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [ { @@ -1721,18 +1841,18 @@ data: ] }, "gridPos": { - "h": 7, + "h": 6, "w": 17, "x": 0, - "y": 38 + "y": 44 }, - "id": 64, + "id": 66, "options": { "legend": { "calcs": [ "min", - "max", - "mean" + "mean", + "max" ], "displayMode": "table", "placement": "bottom", @@ -1748,90 +1868,26 @@ data: "targets": [ { "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "VictoriaLogs" + "type": "prometheus", + "uid": "VictoriaMetrics" }, "editorMode": "code", - "expr": "kubernetes.pod_namespace:\"$namespace\" AND \"RLN proof\"\n| copy _time as start_marker\n| union (\n kubernetes.pod_namespace:\"$namespace\" AND \"proof generated successfully\"\n )\n| replace (\"\\\"\", \"\") at kubernetes.pod_name\n| sort by (kubernetes.pod_name, _time)\n| running_stats by (kubernetes.pod_name) max(start_marker) as latest_start\n| filter \"proof generated successfully\"\n| math (_time - latest_start) / 1000000 as duration_ms\n| sort by (_time) desc\n", - "maxLines": 10000, - "queryType": "instant", + "expr": "sum(rate(mix_rln_proof_generation_duration_seconds_sum{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (instance)\n/\nsum(rate(mix_rln_proof_generation_duration_seconds_count{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (instance)", + "instant": false, + "legendFormat": "__auto", + "range": true, "refId": "A" } ], - "title": "Proof Generation Time", - "transformations": [ - { - "id": "extractFields", - "options": { - "source": "labels" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": { - "Time": true, - "duration_ms": true, - "kubernetes.pod_name": true, - "kubernetes.pod_namespace": true - }, - "indexByName": { - "Time": 0, - "duration_ms": 1, - "kubernetes.pod_name": 3, - "kubernetes.pod_namespace": 2 - }, - "renameByName": { - "duration_ms": "", - "kubernetes.pod_name": "pod", - "kubernetes.pod_namespace": "namespace" - } - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time" - }, - { - "destinationType": "number", - "targetField": "duration_ms" - }, - { - "destinationType": "string", - "targetField": "namespace" - }, - { - "destinationType": "string", - "targetField": "pod" - } - ], - "fields": {} - } - }, - { - "id": "partitionByValues", - "options": { - "fields": [ - "pod", - "namespace" - ], - "keepFields": false - } - } - ], + "title": "Avgerage Proof Generation Duration (Over Time)", "type": "timeseries" }, { "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "VictoriaLogs" + "type": "prometheus", + "uid": "VictoriaMetrics" }, - "description": "Max is 1 hour. Fetching more than last 1 hour is not supported for now.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1851,7 +1907,7 @@ data: } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [ { @@ -1869,14 +1925,14 @@ data: ] }, "gridPos": { - "h": 7, + "h": 6, "w": 7, "x": 17, - "y": 38 + "y": 44 }, - "id": 65, + "id": 67, "options": { - "displayMode": "basic", + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", @@ -1889,9 +1945,7 @@ data: "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "p95" - ], + "calcs": [], "fields": "", "values": false }, @@ -1903,82 +1957,358 @@ data: "targets": [ { "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "VictoriaLogs" + "type": "prometheus", + "uid": "VictoriaMetrics" }, "editorMode": "code", - "expr": "kubernetes.pod_namespace:\"$namespace\" AND \"RLN proof\"\n| copy _time as start_marker\n| union (\n kubernetes.pod_namespace:\"$namespace\" AND \"proof generated successfully\"\n )\n| replace (\"\\\"\", \"\") at kubernetes.pod_name\n| sort by (kubernetes.pod_name, _time)\n| running_stats by (kubernetes.pod_name) max(start_marker) as latest_start\n| filter \"proof generated successfully\"\n| math (_time - latest_start) / 1000000 as duration_ms\n| sort by (_time) desc\n", - "maxLines": 10000, - "queryType": "instant", + "expr": "histogram_quantile(0.95,\n sum(rate(mix_rln_proof_generation_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le, instance)\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, "refId": "A" } ], - "title": "Proof Generation Time (95th Percentile)", - "transformations": [ - { - "id": "extractFields", - "options": { - "source": "labels" + "title": "Proof Generation Duration p95", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 68, + "panels": [], + "title": "RLN Proof Verification", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "description": "per node histogram can be obtained by setting pod selector to a target node instead of All", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ + { + "id": "displayName", + "value": "${__field.labels.pod}" + } + ] } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(mix_rln_proof_verification_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le))", + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" }, { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": { - "Time": true, - "duration_ms": true, - "kubernetes.pod_name": true, - "kubernetes.pod_namespace": true + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(mix_rln_proof_verification_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(mix_rln_proof_verification_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Proof Verification Duration Histogram", + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "description": "SUM/COUNT", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "indexByName": { - "Time": 0, - "duration_ms": 1, - "kubernetes.pod_name": 3, - "kubernetes.pod_namespace": 2 + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "renameByName": { - "duration_ms": "", - "kubernetes.pod_name": "pod", - "kubernetes.pod_namespace": "namespace" + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ + { + "id": "displayName", + "value": "${__field.labels.pod}" + } + ] } + ] + }, + "gridPos": { + "h": 6, + "w": 17, + "x": 0, + "y": 57 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time" - }, + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "sum(rate(mix_rln_proof_verification_duration_seconds_sum{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (instance)\n/\nsum(rate(mix_rln_proof_verification_duration_seconds_count{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (instance)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Avgerage Proof Verification Duration (Over Time)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "destinationType": "number", - "targetField": "duration_ms" + "color": "green", + "value": 0 }, { - "destinationType": "string", - "targetField": "namespace" - }, + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ { - "destinationType": "string", - "targetField": "pod" + "id": "displayName", + "value": "${__field.labels.pod}" } - ], - "fields": {} + ] } + ] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 17, + "y": 57 + }, + "id": 71, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.3.0", + "targets": [ { - "id": "partitionByValues", - "options": { - "fields": [ - "pod", - "namespace" - ], - "keepFields": false - } + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95,\n sum(rate(mix_rln_proof_verification_duration_seconds_bucket{namespace=~\"$namespace\",instance=~\"$pod\"}[$__rate_interval])) by (le, instance)\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" } ], + "title": "Proof Verification Duration p95", "type": "bargauge" } ], @@ -2036,5 +2366,5 @@ data: "timezone": "browser", "title": "Internal - MIX testnet 0.2", "uid": "akgtzjg29062026", - "version": 11 + "version": 12 } \ No newline at end of file diff --git a/fleet/otlp-collector-external/fleet.yaml b/fleet/otlp-collector-external/fleet.yaml index f219e6e..7efac70 100644 --- a/fleet/otlp-collector-external/fleet.yaml +++ b/fleet/otlp-collector-external/fleet.yaml @@ -25,6 +25,18 @@ helm: logsCollection: enabled: false + # (timeoutSeconds: 1, no initialDelaySeconds) proved too tight for this + livenessProbe: + initialDelaySeconds: 20 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + readinessProbe: + initialDelaySeconds: 20 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + service: type: ClusterIP annotations: diff --git a/fleet/otlp-collector/fleet.yaml b/fleet/otlp-collector/fleet.yaml index 25299c0..1af2f5b 100644 --- a/fleet/otlp-collector/fleet.yaml +++ b/fleet/otlp-collector/fleet.yaml @@ -21,6 +21,19 @@ helm: logsCollection: enabled: true + # Chart defaults (timeoutSeconds: 1, no initialDelaySeconds) are too + # tight for this collector + livenessProbe: + initialDelaySeconds: 20 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + readinessProbe: + initialDelaySeconds: 20 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + service: type: ClusterIP diff --git a/fleet/vaclab-ntp/config/config.yaml b/fleet/vaclab-ntp/config/config.yaml new file mode 100644 index 0000000..473a6ad --- /dev/null +++ b/fleet/vaclab-ntp/config/config.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vaclab-ntp-server-conf +data: + chrony.conf: | + server consolidare-continuum.com iburst + server time.cloudflare.com iburst + driftfile /var/lib/chrony/drift + rtcsync + makestep 1.0 3 + local stratum 10 + allow 185.181.228.0/24 + allow 65.21.76.50/32 + allow 10.42.0.0/16 + allow 10.43.0.0/16 + logdir /var/log/chrony \ No newline at end of file diff --git a/fleet/vaclab-ntp/config/fleet.yaml b/fleet/vaclab-ntp/config/fleet.yaml new file mode 100644 index 0000000..404da2b --- /dev/null +++ b/fleet/vaclab-ntp/config/fleet.yaml @@ -0,0 +1 @@ +defaultNamespace: vaclab-ntp \ No newline at end of file diff --git a/fleet/vaclab-ntp/fleet.yaml b/fleet/vaclab-ntp/fleet.yaml new file mode 100644 index 0000000..52b8dcc --- /dev/null +++ b/fleet/vaclab-ntp/fleet.yaml @@ -0,0 +1,4 @@ +defaultNamespace: vaclab-ntp +dependsOn: + - name: vaclab-2-fleet-longhorn + - name: vaclab-2-fleet-storage \ No newline at end of file diff --git a/fleet/vaclab-ntp/server/fleet.yaml b/fleet/vaclab-ntp/server/fleet.yaml new file mode 100644 index 0000000..771e48d --- /dev/null +++ b/fleet/vaclab-ntp/server/fleet.yaml @@ -0,0 +1,3 @@ +defaultNamespace: vaclab-ntp +dependsOn: + - name: vaclab-2-fleet-vaclab-ntp-config \ No newline at end of file diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml new file mode 100644 index 0000000..a18cadc --- /dev/null +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ntp-server +spec: + replicas: 1 + serviceName: ntp-server + selector: + matchLabels: + app: ntp-server + template: + metadata: + labels: + app: ntp-server + spec: + # hostNetwork + SYS_TIME: this pod directly corrects metal-01's + # real kernel clock. Without SYS_TIME, chronyd can only *measure* + # drift and mathematically compensate for it in served responses, but + # never actually correct the underlying clock. + # Because this pod owns metal-01's real port 123 via hostNetwork, + # metal-01 must NOT also run a native host-level chrony client - see + # ansible/inventory.yaml's ntp_server_hosts group. 2 daemons + # disciplining the same kernel clock fight each other and cause instability. + # The host-level chrony client is disabled on metal-01. + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + containers: + - name: ntp-server + resources: + requests: + memory: "1Gi" + cpu: "2" + limits: + memory: "1Gi" + cpu: "2" + image: cturra/ntp:latest + command: ["/bin/sh", "-c"] + args: + - | + if [ -d /run/chrony ]; then + chown -R chrony:chrony /run/chrony + chmod o-rx /run/chrony + rm -f /var/run/chrony/chronyd.pid + fi + if [ -d /var/lib/chrony ]; then + chown -R chrony:chrony /var/lib/chrony + fi + exec /usr/sbin/chronyd -u chrony -d -L 0 + securityContext: + capabilities: + add: ["SYS_TIME", "NET_BIND_SERVICE"] + # SYS_TIME alone isn't enough + seccompProfile: + type: Unconfined + volumeMounts: + - name: conf + mountPath: /etc/chrony/chrony.conf + subPath: chrony.conf + - name: ntp-drift + mountPath: /var/lib/chrony + volumes: + - name: conf + configMap: + name: vaclab-ntp-server-conf + volumeClaimTemplates: + - metadata: + name: ntp-drift + spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn-vaclab + resources: + requests: + storage: 1Gi diff --git a/fleet/vaclab-ntp/server/service.yaml b/fleet/vaclab-ntp/server/service.yaml new file mode 100644 index 0000000..5f67989 --- /dev/null +++ b/fleet/vaclab-ntp/server/service.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Service +metadata: + name: ntp-server +spec: + clusterIP: None + selector: + app: ntp-server