Increase Monitoring Stack Scalability/Resources and Update Ansible

ansible/group_vars/all/vault.yml

@@ -1,58 +1,74 @@
 $ANSIBLE_VAULT;1.1;AES256
-33313338326465616463373739376466643235366232633666393939623432303264353066333761
-6566653435613532366432353739306535353932646435660a373138383639613432386238656530
-66376265303131336263303361323061396134306135303031616432313063663564343835313830
-3236626337666461370a336664663432306663383261366134356366376530363333643864616435
-32343661653638373339613132306335313632363564306534326332316636656135343164373965
-64363339653434363062666539663932653462653036373932343331323230383236313861313466
-30613937386430316236623036313035353437643534386634326139383637363333363039626337
-63303237653838336131356130333337373761663637346332363962316238646565643039653262
-64303662333862666664313433613937666239333331643333646638633632373531303161376162
-66376262346238636361623566386539666561346531396162323761353763656538343134643562
-31306230393165656631366365623733353139383435323564303336383235626465636462663530
-34646430656366656664323838363432336131313333653261613466323435363838616532376332
-66373932656131353331313039396236383762663637616133633134383938613039353231323361
-34383966653333623266316165313431363436363166656431343134353631313564653237366430
-66323039303532626434303839333634616633393733333362656365333465346666636431303537
-63323461313134373665376532656364383664333764643165363530323738616138303338623963
-37346337613933333366653262356233313130626465636365613530396364386666306662333837
-37303361303434313233643034323736343533383739353233356131366531363038623733663466
-31653561633061346239333730623634613438623033383463343064663035366165393531613135
-30363233623932353562313465303233656362386665353164613966316265626234343036396537
-35646637363338363261346434373135363932313166313930623761626231303362393463646162
-35656535333161363765333664356366306436306334666434333136643035633335613666663536
-38363039626136313137383538326230623634613134393365303863336162313562313665313230
-31313036346564633461653633303263383534643933303465633164333261366234616130623464
-30666566333266316631663565326237346561323134616535343730643562316464613133316336
-63393038663031323437323630666435343032393461346331646137376330623333363531383232
-30316663373765333164663832663832643565363466613066613964343837633565353965333966
-62613839643637336437666363613136353337633534633631653630356165613339376165633438
-66623436626239343265636137363432343730356665663539663364386631623037313163313932
-32623766653031383665393232613636373632623136396463646166393138363832383266306130
-39373537623331323666363936343131643662383538383934373937303735633362303132356535
-35303337613035366239323230313635666235646539616563653566326362383166376431656261
-61363833336262633730313333383238383066386438326432353865343366336639353130373461
-36356439373234356666663066663865323565333261643538386534383461396330633532383163
-34393031373366303362333630613339613834336161343337313539626134633663376165396135
-31373963626261396465363337316233646337336530326331363137383336363261386236376161
-37653466363263323131613635653166373331363631633165313661353963396338316262353561
-35393161303138663235646531613830383566653439656433343366333164613338306438636539
-31383535633766666332366362336162316661336439316234653033303933616639386137633834
-63653961623462656636376362613666656336313461316239666132313638323133336634633266
-62656333643331656339396133363730643537623436383166663761663839366466313564666439
-33386434373963663366636665633133613163393139623938316662323832336439343665316639
-63336333323761363162333765393737346533313939623537396538326366623433366165633232
-31373236366165373839323732633034306166353834356662343265376637353466373864356264
-36396430396664313634396630616636356533336332343539373134306561643265616138393335
-62313936306337666631353964333032333734373432303231343334656431356635373235346337
-63666533383261383665363332316236313031643437373761663762336461633066313861653932
-66306639323033643765346265633161353462323638356463613035333463306662616664616430
-61663738336263643264373137303862316661383665613230363932336132393766633537613735
-65613565333661326265343866323365646138343264326662643665373430303236646164316438
-34623161666166343936393363316135343266633331303539656439613234316630383335343338
-32393332353530613863306334383430356238653964303630316333383234663362303836353539
-35386430646230633062666366333862366339393466383234613765616265353463643165303130
-35653935376364653863373565363065336433313566383464303833343231373163386439313861
-39356361653163323830646334626361623130306336643865356539306466393561383462633064
-39663835383533386663363331663739643663336365636665353561386136346634336361363533
-39303466613466366533366663663335396661346363343636366230623431616639
+61396536316562343664623035616633306633613461356535626163626330316262376363363663
+3161306361633133376638653666396431313035363736650a626363326363343461666630326537
+65313632353237326133653638393339313064363961396461393532663733333133356633323936
+6666323066306365660a656639663630633937653961666666333034623864343364393436643135
+37336136663930353662636633633965656366656336356365343438663238633334366134353538
+39653832643165353361653861663035356337393031353862656131633331346566323630646334
+38643261306330333339643130316466303764393638633766613964656131663665323631383862
+33653761646434323637326633373761366461373336653938643839326462316561393938646264
+32616432633565316163366137656436333533343564313932346332363564343965363263653432
+33643632306163663633316261373136383662623830316166363763646162356633393933313065
+65333532306164616137653963653535326364373937666635663665313366653636393465316136
+62393333346434616339653432333437386665303564343439356639363936303832653061616431
+33323537313935393566336436386538343166373665366536383036313339663165633636323364
+34616564656635626136633538383365663235326636633938363034323461396433666436356332
+32303362343932376634363639383137626335633432353238376534346438386430333636396535
+30316363656266616337653266643932646236373936346437393766353032313261653534316366
+37366635306633343937666537663565636465623265666234393133386532396534336532373538
+64346536363538653961626664363265313433643064316262383364646361383331376432363537
+37333635366663363065643331323162313264306636343730386531393161646637373561363964
+37306336336231666333396637633165346666343239373363616431316134333262636634333662
+39353366323562343230626266326237653333393133633866366661653030306639383630643232
+37653963373137366262633032366162396136623862313632616431313530366130343063643164
+31616434653661323233303237373566393165653939666536373165396363396165643833633262
+36623230316630316261366464623937316230353666356135633262616461623934373361656235
+34613033666337386136396661376564386137626232353666326436363162376330393063353034
+63346565333139323330643734363937396164366364383666303235613833393363633363663036
+30393630353163333231623465636430346265343133623232316634323963393663623530623461
+66353539343037303765376162373131333331386562353963626164343531373938343964383232
+32616166376230303539653030633365346136383336313863313339373334313962316666653333
+33386131633830336162356266613035366133366233323336653932613738383032376266636561
+32343632333164653636623632636535663635343433316662353031353730663839363132393232
+64643534646637386434373166663261653635313334656630336538303362663835326232623862
+35396539303962376235343361663664316337616335393464373339656239616337316132613566
+66646533666138313131323536636566613334653462636131383366663131613633666136333237
+38363832613930663133313133356166663337356631303631323132313961326639646336383161
+35383665333835343633373637633937356665353337623137343537303131653930666238386462
+38646164333731343361616632633062333361333365376463343232383964326231363864643331
+37333533643562386136326432386232656465646337326663306539663064336639383337353330
+37393333313232316230633336326235646234356238383438636634393364663834653465303863
+39353264333934303930373166616638616163393365616339356633313264353766303264613539
+31323937326265643532333665343665353261643665663165366130646237656463643939376566
+65646164633562376261663938333161306338333265363663333966323039643938366663346464
+31636165323834303430376238353161326431646663613836373633396330363937326633353238
+66343335356262366434366632363530353737353963333233333136653531646363666265326561
+38383763396161333931666336316232343130393430313836323632643436306666383535623732
+38313239343365303762646562343431353237643036306664653762653037613433303230386565
+62386133323163363532653362626263623165343639323239386131306663343566376132323239
+39653763366130303832326664653266373638343763613235666635343763633466343834643565
+31356532356561396538646531343638336230366363303030333634333563353531666463366563
+62366139343630326636656336376265346435663335663436383465363337376335383935306562
+33306339303533336537323066623366366464333661326533323161316464343765303530616133
+35383431666337623235383664666335666361303531646662633565366161356534316265376562
+34336437636263396335616334323839666561303533336136633130363836323434336632346231
+30616439656363323166643338346165663935356663386434633131366435613361666264633063
+63336531613461313362383439313837376263346532383133643564376265656535663439386561
+33356139346234363234623033643032353738636232633061333936323430303939653832623135
+35313730313565646161313863623331346166353633333336383863643861303561303562363661
+62373332633061663061363062396133623239643965333964366637363363616161643434633235
+61343233323438363036633735306464336166323839613536353739653064666535383961373831
+36363766633833643562313562333863613231613937356266366133353531363266323136616537
+38393565323430613633323030626534643230306339373634623334356461326134613531346639
+62363038373965383533353833633961346363366363626538636166343666366339653239636266
+65316632386266346637376632666663303430376235633762346436623439386637616139376364
+33303438626136333632383033663063383930386438336166393134353264376332316430373034
+61303732316162616266643836393062663063383865656330636364333337396265666665643962
+39656639616533393266303337393364616338616361653462623136663965653264643032663931
+35613563656532323761323566656666313665666561353339633733366233366161313030363138
+39633161656632366237623339356563656339633032396464663234316631613232666438383630
+66343032356661303463633332333065633462373065343464333233386439326435613361343138
+62623535666638316637306230313139353264626532616331366336663435643433313539353563
+38393165303465643266666231653732666436666635303864356463353534323035623130326432
+38623435383038356537666366613830313539383666323266626562383934306131373761306239
+3662

ansible/playbooks/setup_cluster.yaml

@@ -120,6 +120,13 @@
     - name: Import opentelemetry resources tasks
       ansible.builtin.import_tasks: tasks/opentelemetry_resources.yaml
 
+- name: Create dst_dashboard resources
+  hosts: server
+  run_once: true
+  tasks:
+    - name: Import dst_dashboard resources tasks
+      ansible.builtin.import_tasks: tasks/dst_dashboard_resources.yaml
+
 - name: Install Rancher and Fleet
   hosts: server
   run_once: true

ansible/playbooks/tasks/cilium.yaml

@@ -18,6 +18,37 @@
     force_update: true
   when: cilium_installed.rc != 0
 
+# Cilium's cni.chainingMode=portmap (below) requires the standalone "portmap" CNI
+# plugin binary to exist on every node at /opt/cni/bin/
+- name: Check if portmap CNI binary is already installed
+  ansible.builtin.stat:
+    path: /opt/cni/bin/portmap
+  register: portmap_binary
+
+- name: Ensure /opt/cni/bin exists
+  ansible.builtin.file:
+    path: /opt/cni/bin
+    state: directory
+    mode: "0755"
+  when: not portmap_binary.stat.exists
+
+- name: Download containernetworking/plugins release (for the portmap binary)
+  ansible.builtin.get_url:
+    url: https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-amd64-v1.5.1.tgz
+    dest: /tmp/cni-plugins-v1.5.1.tgz
+    mode: "0644"
+  when: not portmap_binary.stat.exists
+
+- name: Install portmap binary into /opt/cni/bin
+  ansible.builtin.unarchive:
+    src: /tmp/cni-plugins-v1.5.1.tgz
+    dest: /opt/cni/bin
+    remote_src: true
+    mode: "0755"
+    include:
+      - portmap
+  when: not portmap_binary.stat.exists
+
 - name: Install Cilium via Helm
   kubernetes.core.helm:
     name: cilium
@@ -29,6 +60,11 @@
     wait_timeout: 10m
     kubeconfig: /etc/rancher/k3s/k3s.yaml
     values:
+      # Chain with the portmap CNI plugin for hostPort support (requires the
+      # portmap binary installed at /opt/cni/bin)
+      cni:
+        chainingMode: portmap
+
       # IPAM configuration for K3s default pod CIDR
       ipam:
         mode: cluster-pool
@@ -51,6 +87,13 @@
       # Network policy enforcement
       policyEnforcementMode: default
 
+      # Raise Cilium's API rate limits above their defaults -- too low for
+      # bursts of ~150-160 simultaneous pod creates/deletes per node
+      extraArgs:
+        - "--api-rate-limit=endpoint-create=rate-limit:20/s,rate-burst:40,parallel-requests:20"
+        - "--api-rate-limit=endpoint-delete=parallel-requests:20"
+        - "--api-rate-limit=endpoint-patch=rate-limit:20/s,rate-burst:40,parallel-requests:20"
+
       # Disable IPv6
       ipv6:
         enabled: false

ansible/playbooks/tasks/dst_dashboard_resources.yaml

@@ -0,0 +1,40 @@
+---
+- name: Create dst-dashboard namespace if not exists
+  kubernetes.core.k8s:
+    state: present
+    kubeconfig: /etc/rancher/k3s/k3s.yaml
+    definition:
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: dst-dashboard
+
+- name: Create dst-dashboard JWT secret
+  kubernetes.core.k8s:
+    state: present
+    kubeconfig: /etc/rancher/k3s/k3s.yaml
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: dst-dashboard-jwt-secret
+        namespace: dst-dashboard
+      type: Opaque
+      stringData:
+        secret: "{{ dst_dashboard_jwt_secret }}"
+
+- name: Create dst mongo secrets
+  kubernetes.core.k8s:
+    state: present
+    kubeconfig: /etc/rancher/k3s/k3s.yaml
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: dst-mongo-prod-credentials
+        namespace: dst-dashboard
+      type: Opaque
+      stringData:
+        password: "{{ dst_mongo_prod_password }}"
+        username: "{{ dst_mongo_prod_username }}"
+        uri: "{{ dst_mongo_prod_uri }}"

fleet/authentik/fleet.yaml

@@ -187,6 +187,16 @@ helm:
         limits:
           cpu: "8"
           memory: "8Gi"
+      # authentik writes its worker heartbeat file via mmap into /dev/shm; the container
+      # default 64Mi tmpfs fills up and workers get failed to proxy to backend: EOF"
+      volumes:
+        - name: authentik-shm
+          emptyDir:
+            medium: Memory
+            sizeLimit: 1Gi
+      volumeMounts:
+        - name: authentik-shm
+          mountPath: /dev/shm
       ingress:
         enabled: true
         ingressClassName: traefik

fleet/dst-dashboard/backend/backend.yaml

@@ -48,6 +48,8 @@ spec:
               key: secret
         - name: DST_CONFIG_PATH
           value: /app/config/config.yaml
+        - name: DST_ALLOWED_ORIGINS
+          value: "https://dashboard.lab.vac.dev,https://dev.dashboard.lab.vac.dev"
         volumeMounts:
         - name: config
           mountPath: /app/config

fleet/monitoring-priority-class/fleet.yaml

@@ -0,0 +1,3 @@
+defaultNamespace: vmetrics
+dependsOn:
+  - name: vaclab-2-fleet-storage

fleet/monitoring-priority-class/priorityclass.yaml

@@ -0,0 +1,11 @@
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: monitoring-critical
+value: 900000000
+globalDefault: false
+preemptionPolicy: PreemptLowerPriority
+description: >-
+  Assigned to victoria-metrics-stack and victoria-logs-cluster components so they
+  keep a guaranteed CPU share and keep higher priority than experiment/simulation
+  workloads under high resource pressure.

fleet/otlp-collector-external/fleet.yaml

@@ -37,6 +37,22 @@ helm:
         cpu: "4"
         memory: 5Gi
 
+    tolerations:
+      - key: "dedicated"
+        operator: "Equal"
+        value: "monitoring"
+        effect: "PreferNoSchedule"
+    affinity:
+      nodeAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: kubernetes.io/hostname
+                  operator: In
+                  values:
+                    - metal-01.he-eu-hel1.misc.vacdst
+
     ingress:
       enabled: true
       ingressClassName: traefik

fleet/otlp-collector/fleet.yaml

@@ -23,6 +23,22 @@ helm:
     service:
       type: ClusterIP
 
+    tolerations:
+      - key: "dedicated"
+        operator: "Equal"
+        value: "monitoring"
+        effect: "PreferNoSchedule"
+    affinity:
+      nodeAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: kubernetes.io/hostname
+                  operator: In
+                  values:
+                    - metal-01.he-eu-hel1.misc.vacdst
+
     ports:
       otlp:
         enabled: true