Deploy Grafana Tempo and Enable Tracing Injection

ansible/group_vars/all/vault.yml

@@ -1,74 +1,74 @@
 $ANSIBLE_VAULT;1.1;AES256
-61396536316562343664623035616633306633613461356535626163626330316262376363363663
-3161306361633133376638653666396431313035363736650a626363326363343461666630326537
-65313632353237326133653638393339313064363961396461393532663733333133356633323936
-6666323066306365660a656639663630633937653961666666333034623864343364393436643135
-37336136663930353662636633633965656366656336356365343438663238633334366134353538
-39653832643165353361653861663035356337393031353862656131633331346566323630646334
-38643261306330333339643130316466303764393638633766613964656131663665323631383862
-33653761646434323637326633373761366461373336653938643839326462316561393938646264
-32616432633565316163366137656436333533343564313932346332363564343965363263653432
-33643632306163663633316261373136383662623830316166363763646162356633393933313065
-65333532306164616137653963653535326364373937666635663665313366653636393465316136
-62393333346434616339653432333437386665303564343439356639363936303832653061616431
-33323537313935393566336436386538343166373665366536383036313339663165633636323364
-34616564656635626136633538383365663235326636633938363034323461396433666436356332
-32303362343932376634363639383137626335633432353238376534346438386430333636396535
-30316363656266616337653266643932646236373936346437393766353032313261653534316366
-37366635306633343937666537663565636465623265666234393133386532396534336532373538
-64346536363538653961626664363265313433643064316262383364646361383331376432363537
-37333635366663363065643331323162313264306636343730386531393161646637373561363964
-37306336336231666333396637633165346666343239373363616431316134333262636634333662
-39353366323562343230626266326237653333393133633866366661653030306639383630643232
-37653963373137366262633032366162396136623862313632616431313530366130343063643164
-31616434653661323233303237373566393165653939666536373165396363396165643833633262
-36623230316630316261366464623937316230353666356135633262616461623934373361656235
-34613033666337386136396661376564386137626232353666326436363162376330393063353034
-63346565333139323330643734363937396164366364383666303235613833393363633363663036
-30393630353163333231623465636430346265343133623232316634323963393663623530623461
-66353539343037303765376162373131333331386562353963626164343531373938343964383232
-32616166376230303539653030633365346136383336313863313339373334313962316666653333
-33386131633830336162356266613035366133366233323336653932613738383032376266636561
-32343632333164653636623632636535663635343433316662353031353730663839363132393232
-64643534646637386434373166663261653635313334656630336538303362663835326232623862
-35396539303962376235343361663664316337616335393464373339656239616337316132613566
-66646533666138313131323536636566613334653462636131383366663131613633666136333237
-38363832613930663133313133356166663337356631303631323132313961326639646336383161
-35383665333835343633373637633937356665353337623137343537303131653930666238386462
-38646164333731343361616632633062333361333365376463343232383964326231363864643331
-37333533643562386136326432386232656465646337326663306539663064336639383337353330
-37393333313232316230633336326235646234356238383438636634393364663834653465303863
-39353264333934303930373166616638616163393365616339356633313264353766303264613539
-31323937326265643532333665343665353261643665663165366130646237656463643939376566
-65646164633562376261663938333161306338333265363663333966323039643938366663346464
-31636165323834303430376238353161326431646663613836373633396330363937326633353238
-66343335356262366434366632363530353737353963333233333136653531646363666265326561
-38383763396161333931666336316232343130393430313836323632643436306666383535623732
-38313239343365303762646562343431353237643036306664653762653037613433303230386565
-62386133323163363532653362626263623165343639323239386131306663343566376132323239
-39653763366130303832326664653266373638343763613235666635343763633466343834643565
-31356532356561396538646531343638336230366363303030333634333563353531666463366563
-62366139343630326636656336376265346435663335663436383465363337376335383935306562
-33306339303533336537323066623366366464333661326533323161316464343765303530616133
-35383431666337623235383664666335666361303531646662633565366161356534316265376562
-34336437636263396335616334323839666561303533336136633130363836323434336632346231
-30616439656363323166643338346165663935356663386434633131366435613361666264633063
-63336531613461313362383439313837376263346532383133643564376265656535663439386561
-33356139346234363234623033643032353738636232633061333936323430303939653832623135
-35313730313565646161313863623331346166353633333336383863643861303561303562363661
-62373332633061663061363062396133623239643965333964366637363363616161643434633235
-61343233323438363036633735306464336166323839613536353739653064666535383961373831
-36363766633833643562313562333863613231613937356266366133353531363266323136616537
-38393565323430613633323030626534643230306339373634623334356461326134613531346639
-62363038373965383533353833633961346363366363626538636166343666366339653239636266
-65316632386266346637376632666663303430376235633762346436623439386637616139376364
-33303438626136333632383033663063383930386438336166393134353264376332316430373034
-61303732316162616266643836393062663063383865656330636364333337396265666665643962
-39656639616533393266303337393364616338616361653462623136663965653264643032663931
-35613563656532323761323566656666313665666561353339633733366233366161313030363138
-39633161656632366237623339356563656339633032396464663234316631613232666438383630
-66343032356661303463633332333065633462373065343464333233386439326435613361343138
-62623535666638316637306230313139353264626532616331366336663435643433313539353563
-38393165303465643266666231653732666436666635303864356463353534323035623130326432
-38623435383038356537666366613830313539383666323266626562383934306131373761306239
-3662
+38326431636133353432623966323231373136306630353632373062633465353435353234653437
+6364333630633536393863313266323033623137306632660a386662366234653763336563396662
+64663935383362636238313138656634353836663830626366303236363965643831663530633237
+6462653033636331630a663637303430616333393561376431333037616538663264623830323634
+63366436363066366366373130663935356664623363306134323238346666626564383635643539
+32336261373664333232386561663665356433643332323965616538326130626136663666353963
+63663036313463666665353636383336616665383532313264646563366131616337313330333537
+32653235663033386139303339303732643461343362323039626231643936396531333833313236
+61613533356239343433613032663934346536343335396365323162333336366133376461383366
+38333234313435626337356534306262353038303539353539626535306239323336633465646164
+32376530306335323262633166336361653731346533333938666334313637353632363835306565
+65623838313131343738626530626566656136633966366465306236393566653234356463386136
+34333631626435343062373936656237343933343835626332383865626664653062303666316564
+38343663613036353335666631653765396135653431376231613236373166313137326237633263
+62636137393463653937353661323761326431306536376464333833616338333965623833643733
+63656464356337353339323033623864393233376131656232353737376363353231313832303731
+31666239353431653132316238383338386561616137633163343465633333366534663364373930
+33373865393838386237633838663238613862383961633635623730313735383461666663326632
+62653162376362666135383836316261323762323462316561316639623437633636623833353961
+39353965306231646137613530373866663137346139633934666134363936623631353735316238
+36346437323163326631643233643330646166303931313534636336376562653463393162306530
+35383530663662613433653634613538376362666232393334346262376261663432323062613935
+66383237393133643034356234633339326162616164653366383266643634326161643934666538
+31643830653435613237666332623465653835616230366465356335303633393464383661656463
+66333438353639386363656233383833333565363834346237633562343964623533616537663061
+34366636356132343932613234363332646331376639343930653335663636626636343235376532
+33663636343237376632613265363966636238326438643034346335643463666435643564646639
+66633664343030303537393661386262656238666662633464373031393166623835323966373839
+36616164646535393031616236373866313935343133643538616537303966663133346564383836
+61616566633066363465656435626137623936353934373135646635336139613031373337623961
+35396436656435616134663733646235633531303532343965333336393931336566383362383830
+39383861363263643464303363393737633363323032663635636663366562643463316531346530
+37653864316663363864653266663638323032313765646338323435363934643237363863396435
+39643636393061613463323365623538666335313931613361323438326530303637303632353661
+30653465363339323636313739306635346263653161356234366334323866393132363939306665
+63626563666537383566616631373535643465343130356537383633303334373134323633343135
+38313331393732366564373565666566643037393464386432323939333430626361333931376230
+31653637643838396232393563633134323164336566646536613930656332333963336430313238
+39626332356537313638626131386265343733373035663930656561666533356565313735373330
+32663561653365633636326636303735393031656630353961613234633732303061306431313738
+63303462353531336164656131376266303363636433613837636465363361323766386464613330
+66646332386634386236333039646339343833353930386361366265333531333162333036633165
+65613731386666653962313432396261383665616161313762343537646362333032363231356461
+66303365613434663932626534393539643263636662666531323833666565346130336661356630
+39623439616161613637623335303165343838386132323237323032393064663332333666636132
+66343966646437376536396136626235306433393966636164646463653161646164663730383536
+64383765626633643738383234663661316636356632353766653834303839303430303363333964
+66363632373364613338333237616230653161646561386332353738326235353763363334666635
+62383561633238373334633635643661626335383336663331633562303734616563303533666361
+32636438623730363163613364623061613930633433623866633431623862373439326133303639
+31356466323231326461313933383262646263626365326437333261326339633761623939386136
+30333935336139386131303536333738646264623834643836346139656431633837336461383532
+30393761636237323062353635356265646335656131363532393161616435623463626631363435
+32306261383837653836343436383762653932326232383465316138353761353966303033383365
+65376666626564326237373763656362616438636139363331356530383139366364363934363563
+62616464333035376638383364613735363438326565386566633834323433383961636332633330
+66343733643131643437306135646137323762363463653432646166383166356237323663653564
+35343265386664666139343038376437633965623266333532373539663432313930373937626338
+38613234356130643831393561326635613035343163613232633263306661393035373466636433
+37396565633963633038656166376561346566663531333262373163653236303164363464396364
+63633135393264346262393031313430343164623532386662333166623766393931313962366132
+32326638346335323930396634636339643532666632313332616234383662343832333437396234
+30326239396564623062643632316461373433323463373765396638376430663161643461373137
+34376236363864363833393435326335653262643830646432636165376630616464646532393063
+32386532383634326166636563616366663263353132323539303137623465373065366335356635
+61623562366338353362323136623430616365666563336230323464306264316263356135623139
+38616233393466386235636433383164383735326365323131383164613962653862343437363336
+66646134303233373134643063356331636132323732343539643333326134343733376263616364
+61386536376235366136333034353964316433313833346137613437353064646231396564366539
+39653965326636383339653639343034353139613565323332316534643236323239643632343561
+33306633363765383161646461303862383835643363386331663830373431386434333339343161
+61393238373835343038633064643333373130623734366366376132333366326563393236633338
+64663938343066396436356435333632636530313861346438626233626435626239

extras/otlp-trace-test-job.yaml

@@ -0,0 +1,32 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: otlp-trace-test
+  namespace: opentelemetry
+spec:
+  backoffLimit: 1
+  ttlSecondsAfterFinished: 300
+  template:
+    metadata:
+      labels:
+        app: otlp-trace-test
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: telemetrygen
+          image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.146.0
+          env:
+            - name: OTEL_CLIENT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: otel-client-token
+                  key: token
+          args:
+            - "traces"
+            - "--otlp-endpoint=otlp.lab.vac.dev:30317"
+            - "--otlp-insecure"
+            - "--otlp-header"
+            - "Authorization=\"Bearer $(OTEL_CLIENT_TOKEN)\""
+            - "--traces=5"
+            - "--otlp-attributes"
+            - "service.name=\"otlp-trace-test\""

fleet/grafana-tempo-external/fleet.yaml

@@ -0,0 +1,46 @@
+defaultNamespace: grafana-tempo-external
+dependsOn:
+  - name: vaclab-2-fleet-longhorn
+  - name: vaclab-2-fleet-storage
+
+helm:
+  repo: https://grafana.github.io/helm-charts
+  chart: tempo
+  version: 1.24.4
+  releaseName: tempo-external
+  takeOwnership: true
+  values:
+    tempo:
+      retention: 4380h # 6 months
+      resources:
+        requests:
+          cpu: "2"
+          memory: 4Gi
+        limits:
+          cpu: "2"
+          memory: 8Gi
+      # otlp grpc (4317) and http (4318) receivers are enabled by default,
+      # matching the ports the otlp-collector-external already exports on.
+
+    persistence:
+      enabled: true
+      storageClassName: longhorn-vaclab
+      size: 100Gi
+
+    priorityClassName: monitoring-critical
+
+    tolerations:
+      - key: "dedicated"
+        operator: "Equal"
+        value: "monitoring"
+        effect: "PreferNoSchedule"
+    affinity:
+      nodeAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: kubernetes.io/hostname
+                  operator: In
+                  values:
+                    - metal-01.he-eu-hel1.misc.vacdst

fleet/grafana-tempo/fleet.yaml

@@ -0,0 +1,47 @@
+defaultNamespace: grafana-tempo
+dependsOn:
+  - name: vaclab-2-fleet-longhorn
+  - name: vaclab-2-fleet-storage
+  - name: vaclab-2-fleet-monitoring-priority-class
+
+helm:
+  repo: https://grafana.github.io/helm-charts
+  chart: tempo
+  version: 1.24.4
+  releaseName: tempo
+  takeOwnership: true
+  values:
+    tempo:
+      retention: 4380h # 6 months
+      resources:
+        requests:
+          cpu: "2"
+          memory: 4Gi
+        limits:
+          cpu: "2"
+          memory: 8Gi
+      # otlp grpc (4317) and http (4318) receivers are enabled by default,
+      # matching the ports the otlp-collector already exports on.
+
+    persistence:
+      enabled: true
+      storageClassName: longhorn-vaclab
+      size: 100Gi
+
+    priorityClassName: monitoring-critical
+
+    tolerations:
+      - key: "dedicated"
+        operator: "Equal"
+        value: "monitoring"
+        effect: "PreferNoSchedule"
+    affinity:
+      nodeAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: kubernetes.io/hostname
+                  operator: In
+                  values:
+                    - metal-01.he-eu-hel1.misc.vacdst

fleet/otlp-collector-external/fleet.yaml

@@ -1,6 +1,7 @@
 defaultNamespace: opentelemetry
 dependsOn:
   - name: vaclab-2-fleet-victoria-metrics-stack
+  - name: vaclab-2-fleet-grafana-tempo-external
 
 helm:
   repo: https://open-telemetry.github.io/opentelemetry-helm-charts/
@@ -166,6 +167,15 @@ helm:
                 - set(attributes["pod_uid"], attributes["k8s.pod.uid"])
                 - set(attributes["pod_ip"], attributes["k8s.pod.ip"])
                 - set(attributes["node"], attributes["k8s.node.name"])
+          trace_statements:
+            - context: resource
+              statements:
+                - set(attributes["dst_source"], "external")
+                - set(attributes["namespace"], attributes["k8s.namespace.name"])
+                - set(attributes["pod"], attributes["k8s.pod.name"])
+                - set(attributes["pod_uid"], attributes["k8s.pod.uid"])
+                - set(attributes["pod_ip"], attributes["k8s.pod.ip"])
+                - set(attributes["node"], attributes["k8s.node.name"])
 
       receivers:
         otlp:
@@ -188,6 +198,10 @@ helm:
           logs_endpoint: http://vlc-external-victoria-logs-cluster-vlinsert.victorialogs-external.svc.cluster.local:9481/insert/opentelemetry/v1/logs
           tls:
             insecure: true
+        otlp/tempo:
+          endpoint: tempo-external.grafana-tempo-external.svc.cluster.local:4317
+          tls:
+            insecure: true
 
       service:
         extensions: [health_check, bearertokenauth]
@@ -199,4 +213,8 @@ helm:
           metrics:
             receivers: [otlp]
             processors: [memory_limiter, k8sattributes, transform, deltatocumulative]
-            exporters: [otlphttp/victoriametrics]
+            exporters: [otlphttp/victoriametrics]
+          traces:
+            receivers: [otlp]
+            processors: [memory_limiter, k8sattributes, transform]
+            exporters: [otlp/tempo]

fleet/otlp-collector/fleet.yaml

@@ -1,6 +1,7 @@
 defaultNamespace: opentelemetry
 dependsOn:
   - name: vaclab-2-fleet-victoria-metrics-stack
+  - name: vaclab-2-fleet-grafana-tempo
 
 helm:
   repo: https://open-telemetry.github.io/opentelemetry-helm-charts/
@@ -115,6 +116,14 @@ helm:
                 - set(attributes["pod_uid"], attributes["k8s.pod.uid"])
                 - set(attributes["pod_ip"], attributes["k8s.pod.ip"])
                 - set(attributes["node"], attributes["k8s.node.name"])
+          trace_statements:
+            - context: resource
+              statements:
+                - set(attributes["namespace"], attributes["k8s.namespace.name"])
+                - set(attributes["pod"], attributes["k8s.pod.name"])
+                - set(attributes["pod_uid"], attributes["k8s.pod.uid"])
+                - set(attributes["pod_ip"], attributes["k8s.pod.ip"])
+                - set(attributes["node"], attributes["k8s.node.name"])
 
       receivers:
         otlp:
@@ -136,6 +145,10 @@ helm:
           logs_endpoint: http://vlc-victoria-logs-cluster-vlinsert.victorialogs.svc.cluster.local:9481/insert/opentelemetry/v1/logs
           tls:
             insecure: true
+        otlp/tempo:
+          endpoint: tempo.grafana-tempo.svc.cluster.local:4317
+          tls:
+            insecure: true
 
       service:
         extensions: [health_check, bearertokenauth]
@@ -147,4 +160,8 @@ helm:
           metrics:
             receivers: [otlp]
             processors: [k8sattributes, transform, deltatocumulative]
-            exporters: [otlphttp/victoriametrics]
+            exporters: [otlphttp/victoriametrics]
+          traces:
+            receivers: [otlp]
+            processors: [k8sattributes, transform]
+            exporters: [otlp/tempo]