Files
nexus/sreweekly/articles/417/08-distributed-tracing-guide-quick-overview.html
2026-09-12 17:23:01 +08:00

208 lines
150 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en" class="h-full __variable_f367f3 __variable_b436a8 __variable_ce197d __variable_cfa357"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" href="/_next/static/media/a8803dd9fdd15a53-s.p.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" href="/_next/static/media/e244d3ca994d3054-s.p.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" href="/_next/static/media/e4af272ccee01ff0-s.p.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" href="/_next/static/media/f3f7e95f2dbc4fe4-s.p.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" as="image" href="/static/images/Metoro_16_9.svg" fetchPriority="high"/><link rel="stylesheet" href="/_next/static/css/ba854656cbf121ba.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/960e6a5fafc45966.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/65af9316bbe810e3.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/bfde9e8e18a2a61f.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/7c28a0e1df995273.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/b5291668b3a6ced6.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-491b48b36c258e17.js"/><script src="/_next/static/chunks/5594df5f-703d7611243ec4a0.js" async=""></script><script src="/_next/static/chunks/1355-cdc5813df6552503.js" async=""></script><script src="/_next/static/chunks/main-app-679d484c0b54fe7f.js" async=""></script><script src="/_next/static/chunks/d9202a3a-6c4efff19547c375.js" async=""></script><script src="/_next/static/chunks/fbd51a90-4c969d592a1e29cb.js" async=""></script><script src="/_next/static/chunks/6159-7d1f9e865c002a7f.js" async=""></script><script src="/_next/static/chunks/2707-8248e5f0f3ae475e.js" async=""></script><script src="/_next/static/chunks/7918-35abfc54082c840d.js" async=""></script><script src="/_next/static/chunks/9479-bbef253cea568311.js" async=""></script><script src="/_next/static/chunks/6818-b595c894fced36f8.js" async=""></script><script src="/_next/static/chunks/1740-64df73f896fd8888.js" async=""></script><script src="/_next/static/chunks/7651-9c1698f45dc2ca4f.js" async=""></script><script src="/_next/static/chunks/3310-8b48b40a4aa05b46.js" async=""></script><script src="/_next/static/chunks/2751-e2017637308c8f9c.js" async=""></script><script src="/_next/static/chunks/app/layout-46f53436efc58821.js" async=""></script><script src="/_next/static/chunks/9587-30e87c7f19bde82a.js" async=""></script><script src="/_next/static/chunks/4136-66f49c1e9aae6bbc.js" async=""></script><script src="/_next/static/chunks/5319-4ce901258ca1d364.js" async=""></script><script src="/_next/static/chunks/app/blog/%5B...slug%5D/page-8cfd0cc158c6d2e7.js" async=""></script><title>Distributed Tracing: A Whistle Stop Tour</title><meta name="description" content="Know enough to be dangerous in 10 minutes"/><meta name="author" content="Chris Battarbee"/><meta name="keywords" content="observability,kubernetes,ebpf,monitoring,platform"/><meta name="robots" content="index, follow"/><link rel="canonical" href="https://metoro.io/blog/distributed-tracing-whistle-stop-tour"/><link rel="alternate" hrefLang="en" href="https://metoro.io/blog/distributed-tracing-whistle-stop-tour"/><meta property="og:title" content="Distributed Tracing: A Whistle Stop Tour"/><meta property="og:description" content="Know enough to be dangerous in 10 minutes"/><meta property="og:url" content="https://metoro.io/blog/distributed-tracing-whistle-stop-tour"/><meta property="og:image" content="https://metoro.io/static/images/logos/MetoroLogo.png"/><meta property="og:image:width" content="1250"/><meta property="og:image:height" content="1250"/><meta property="og:image:alt" content="Distributed Tracing: A Whistle Stop Tour"/><meta property="og:type" content="article"/><meta property="article:published_time" content="2024-03-13"/><meta property="article:modified_time" content="2024-03-13"/><meta property="article:author" content="Chris Battarbee"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:site" content="@metoroio"/><meta name="twitter:title" content="Distributed Tracing: A Whistle Stop Tour"/><meta name="twitter:description" content="Know enough to be dangerous in 10 minutes"/><meta name="twitter:image" content="https://metoro.io/static/images/logos/MetoroLogo.png"/><link rel="icon" href="/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script type="application/ld+json">{"@context":"https://schema.org","@type":"WebSite","name":"Metoro AI SRE","url":"https://metoro.io","publisher":{"@type":"Organization","name":"Metoro AI SRE","logo":{"@type":"ImageObject","url":"https://metoro.io/static/images/Metoro_16_9.svg"}}}</script><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body class="flex min-h-full bg-[#05080d]"><script>(self.__next_s=self.__next_s||[]).push([0,{"children":"\n if ('scrollRestoration' in history) {\n history.scrollRestoration = 'manual';\n }\n window.scrollTo(0, 0);\n ","id":"scroll-restoration"}])</script><div class="flex w-full flex-col"><div class="HomePage_headerTheme___96CB sticky z-50" style="top:var(--incident-banner-height, 0px)"><header class="relative z-50 border-b border-white/10 bg-[rgba(10,10,10,0.72)] backdrop-blur-[14px]"><div class="mx-auto flex h-16 max-w-[1240px] items-center gap-7 px-4 sm:px-6 lg:px-8"><a href="/" class="flex items-center gap-2"><img alt="Metoro" fetchPriority="high" width="120" height="68" decoding="async" data-nimg="1" class="h-[40px] w-auto" style="color:transparent" src="/static/images/Metoro_16_9.svg"/></a><nav class="hidden flex-1 items-center gap-1 lg:flex"><div class="group relative"><button type="button" class="inline-flex appearance-none items-center gap-1 rounded-[8px] border-0 bg-transparent px-3 py-2 text-[14px] font-normal text-[#ededed] transition-colors hover:bg-white/5 hover:text-white">Product<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" style="transform:none;transition:transform 180ms"><path d="m6 9 6 6 6-6"></path></svg></button><div class="fixed inset-x-4 top-[calc(var(--incident-banner-height,0px)_+_64px)] mx-auto max-w-[1100px] pt-3 hidden"><div class="HomePage_megaMenu__2coTy grid max-h-[calc(100dvh_-_var(--incident-banner-height,0px)_-_96px)] grid-cols-[minmax(0,1fr)_minmax(0,1fr)_260px] gap-5 overflow-y-auto p-5 xl:grid-cols-[minmax(0,1fr)_minmax(0,1fr)_280px] xl:gap-10 xl:p-7"><div><div class="HomePage_eyebrow__Gy9YW mb-4">Kubernetes monitoring</div><div class="flex flex-col gap-2"><a href="/kubernetes-monitoring" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M3 13.125C3 12.504 3.504 12 4.125 12h2.25c.621 0 1.125.504 1.125 1.125v6.75C7.5 20.496 6.996 21 6.375 21h-2.25A1.125 1.125 0 013 19.875v-6.75ZM9.75 8.625c0-.621.504-1.125 1.125-1.125h2.25c.621 0 1.125.504 1.125 1.125v11.25c0 .621-.504 1.125-1.125 1.125h-2.25a1.125 1.125 0 01-1.125-1.125V8.625ZM16.5 4.125c0-.621.504-1.125 1.125-1.125h2.25C20.496 3 21 3.504 21 4.125v15.75c0 .621-.504 1.125-1.125 1.125h-2.25a1.125 1.125 0 01-1.125-1.125V4.125Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Kubernetes monitoring</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">All-in-one Kubernetes observability</span></span></div></a><div class="HomePage_eyebrow__Gy9YW px-3 pb-1 pt-3 text-[10px]">Kubernetes monitoring capabilities</div><a href="/features/kubernetes-apm-application-performance-monitoring" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M3 13.125C3 12.504 3.504 12 4.125 12h2.25c.621 0 1.125.504 1.125 1.125v6.75C7.5 20.496 6.996 21 6.375 21h-2.25A1.125 1.125 0 013 19.875v-6.75ZM9.75 8.625c0-.621.504-1.125 1.125-1.125h2.25c.621 0 1.125.504 1.125 1.125v11.25c0 .621-.504 1.125-1.125 1.125h-2.25a1.125 1.125 0 01-1.125-1.125V8.625ZM16.5 4.125c0-.621.504-1.125 1.125-1.125h2.25C20.496 3 21 3.504 21 4.125v15.75c0 .621-.504 1.125-1.125 1.125h-2.25a1.125 1.125 0 01-1.125-1.125V4.125Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Kubernetes APM</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Dashboards, service maps, and traces</span></span></div></a><a href="/features/ai-agent-monitoring" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M2.036 12.322a1.012 1.012 0 010-.639C3.423 7.51 7.36 4.5 12 4.5c4.638 0 8.573 3.007 9.963 7.178.07.207.07.431 0 .639C20.577 16.49 16.64 19.5 12 19.5c-4.638 0-8.573-3.007-9.964-7.178Z"></path><path stroke-linecap="round" stroke-linejoin="round" d="M15 12a3 3 0 11-6 0 3 3 0 016 0Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">AI Agent Monitoring<span class="HomePage_menuTag__LZjyF">NEW</span></span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Kernel-level agent telemetry</span></span></div></a><a href="/features/kubernetes-infrastructure-monitoring" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M5.25 14.25h13.5m-13.5 0a3 3 0 01-3-3m3 3a3 3 0 100 6h13.5a3 3 0 100-6m-16.5-3a3 3 0 013-3h13.5a3 3 0 013 3m-19.5 0a4.5 4.5 0 01.9-2.7L5.737 5.1a3.375 3.375 0 012.7-1.35h7.126c1.062 0 2.062.5 2.7 1.35l2.587 3.45a4.5 4.5 0 01.9 2.7m0 0a3 3 0 01-3 3m0 3h.008v.008h-.008v-.008Zm0-6h.008v.008h-.008v-.008Zm-3 6h.008v.008h-.008v-.008Zm0-6h.008v.008h-.008v-.008Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Infrastructure Monitoring</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Monitor nodes and clusters</span></span></div></a><a href="/features/kubernetes-logging" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M12 7.5h1.5m-1.5 3h1.5m-7.5 3h7.5m-7.5 3h7.5m3-9h3.375c.621 0 1.125.504 1.125 1.125V18a2.25 2.25 0 01-2.25 2.25M16.5 7.5V18a2.25 2.25 0 002.25 2.25M16.5 7.5V4.875c0-.621-.504-1.125-1.125-1.125H4.125C3.504 3.75 3 4.254 3 4.875V18a2.25 2.25 0 002.25 2.25h13.5M6 7.5h3v3H6v-3Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Log Management</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Collect and analyze logs</span></span></div></a><a href="/features/kubernetes-profiling" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M3.75 13.5 14.25 2.25 12 10.5h8.25L9.75 21.75 12 13.5H3.75Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Container Profiling</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Deep performance insights</span></span></div></a><a href="/features/kubernetes-dashboarding-and-metrics" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M10.5 6a7.5 7.5 0 107.5 7.5h-7.5V6Z"></path><path stroke-linecap="round" stroke-linejoin="round" d="M13.5 10.5H21A7.5 7.5 0 0013.5 3v7.5Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Custom Dashboards</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Metrics and visualizations</span></span></div></a><a href="/features/kubernetes-cost-monitoring" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M21 12V8.25A2.25 2.25 0 0018.75 6H5.25A2.25 2.25 0 003 8.25v7.5A2.25 2.25 0 005.25 18h13.5A2.25 2.25 0 0021 15.75V12Zm0 0h-3.75A1.5 1.5 0 0015.75 13.5v0a1.5 1.5 0 001.5 1.5H21M7.5 6V4.875A1.875 1.875 0 019.375 3h8.25"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Cost Monitoring</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Allocate spend and find waste</span></span></div></a><a href="/features/uptime-monitoring" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M9 12.75 11.25 15 15 9.75m-3-7.036A11.959 11.959 0 013.598 6 11.99 11.99 0 003 9.749c0 5.592 3.824 10.29 9 11.623 5.176-1.332 9-6.03 9-11.622 0-1.31-.21-2.571-.598-3.751h-.152c-3.196 0-6.1-1.248-8.25-3.285Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Uptime Monitoring</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Health checks &amp; status pages</span></span></div></a><a href="/features/kubernetes-cron-job-monitoring" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M6.75 3v2.25M17.25 3v2.25M3.75 9.75h16.5M5.25 21h13.5A2.25 2.25 0 0021 18.75V8.25A2.25 2.25 0 0018.75 6H5.25A2.25 2.25 0 003 8.25v10.5A2.25 2.25 0 005.25 21Z"></path><path stroke-linecap="round" stroke-linejoin="round" d="M12 12.75v2.25l1.5 1.5"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Cron Job Monitoring</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Catch missed and failed runs</span></span></div></a></div></div><div><div class="HomePage_eyebrow__Gy9YW mb-4">AI SRE</div><div class="flex flex-col gap-2"><a href="/ai-sre-agent" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M9.813 15.904 9 18.75l-.813-2.846a4.5 4.5 0 00-3.09-3.09L2.25 12l2.846-.813a4.5 4.5 0 003.09-3.09L9 5.25l.813 2.846a4.5 4.5 0 003.09 3.09L15.75 12l-2.846.813a4.5 4.5 0 00-3.09 3.09ZM18.259 8.715 18 9.75l-.259-1.035a3.375 3.375 0 00-2.455-2.456L14.25 6l1.036-.259a3.375 3.375 0 002.455-2.456L18 2.25l.259 1.035a3.375 3.375 0 002.456 2.456L21.75 6l-1.035.259a3.375 3.375 0 00-2.456 2.456ZM16.894 20.567 16.5 21.75l-.394-1.183a2.25 2.25 0 00-1.423-1.423L13.5 18.75l1.183-.394a2.25 2.25 0 001.423-1.423l.394-1.183.394 1.183a2.25 2.25 0 001.423 1.423l1.183.394-1.183.394a2.25 2.25 0 00-1.423 1.423Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">AI SRE<span class="HomePage_menuTag__LZjyF">NEW</span></span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Autonomous SRE agent for Kubernetes</span></span></div></a><div class="HomePage_eyebrow__Gy9YW px-3 pb-1 pt-3 text-[10px]">AI SRE capabilities</div><a href="/features/ai-root-cause-analysis" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M21 21l-5.197-5.197m0 0A7.5 7.5 0 105.196 5.196a7.5 7.5 0 0010.607 10.607z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Autonomous Issue Detection and RCA</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Automatically detect and resolve incidents</span></span></div></a><a href="/features/ai-deployment-verification" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M9 12.75 11.25 15 15 9.75M21 12c0 1.268-.63 2.39-1.593 3.068a3.745 3.745 0 01-1.043 3.296 3.745 3.745 0 01-3.296 1.043A3.745 3.745 0 0112 21c-1.268 0-2.39-.63-3.068-1.593a3.746 3.746 0 01-3.296-1.043 3.745 3.745 0 01-1.043-3.296A3.745 3.745 0 013 12c0-1.268.63-2.39 1.593-3.068a3.745 3.745 0 011.043-3.296 3.746 3.746 0 013.296-1.043A3.746 3.746 0 0112 3c1.268 0 2.39.63 3.068 1.593a3.746 3.746 0 013.296 1.043 3.746 3.746 0 011.043 3.296A3.745 3.745 0 0121 12Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Deployment Verification<span class="HomePage_menuTag__LZjyF">NEW</span></span><span class="mt-1 block text-[12px] text-[#a1a1a1]">AI-verified deployments</span></span></div></a><a href="/features/ai-alert-investigation" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M14.857 17.082a23.848 23.848 0 005.454-1.31A8.967 8.967 0 0118 9.75v-.7V9a6 6 0 10-12 0v.75a8.967 8.967 0 01-2.312 6.022c1.733.64 3.56 1.085 5.455 1.31m5.714 0a24.255 24.255 0 01-5.714 0m5.714 0a3 3 0 11-5.714 0"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">AI Alert Investigation</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Investigate every alert with AI</span></span></div></a></div></div><div class="border-l border-white/10 pl-6"><div><div class="HomePage_eyebrow__Gy9YW mb-4">Deployment Options</div><div class="flex flex-col gap-2"><a href="/deployment/metoro-cloud" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M2.25 15a4.5 4.5 0 004.5 4.5H18a3.75 3.75 0 001.332-7.257 3 3 0 00-3.758-3.848 5.25 5.25 0 00-10.233 2.33A4.502 4.502 0 002.25 15Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Metoro Cloud</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Fully managed by us</span></span></div></a><a href="/deployment/metoro-byoc" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M12 16.5V9.75m0 0 3 3m-3-3-3 3M6.75 19.5a4.5 4.5 0 01-1.41-8.775 5.25 5.25 0 0110.233-2.33 3 3 0 013.758 3.848A3.752 3.752 0 0118 19.5H6.75Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Metoro BYOC</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Hosted by you, managed by us</span></span></div></a><a href="/deployment/metoro-onprem" class="HomePage_menuItem__zs1Ku"><div class="flex gap-3"><span class="mt-0.5 inline-flex h-9 w-9 shrink-0 items-center justify-center rounded-[10px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M8.25 3v1.5M4.5 8.25H3m18 0h-1.5M4.5 12H3m18 0h-1.5m-15 3.75H3m18 0h-1.5M8.25 19.5V21M12 3v1.5m0 15V21m3.75-18v1.5m0 15V21m-9-1.5h10.5a2.25 2.25 0 002.25-2.25V6.75a2.25 2.25 0 00-2.25-2.25H6.75A2.25 2.25 0 004.5 6.75v10.5a2.25 2.25 0 002.25 2.25Zm.75-12h9v9h-9v-9Z"></path></svg></span><span class="min-w-0"><span class="flex items-center gap-2 text-[14px] font-medium text-white">Metoro On-Prem</span><span class="mt-1 block text-[12px] text-[#a1a1a1]">Airgapped or not, complete isolation</span></span></div></a></div></div></div></div></div></div><a href="https://demo.us-east.metoro.io" class="rounded-[8px] px-3 py-2 text-[14px] text-[#a1a1a1] transition-colors hover:bg-white/5 hover:text-white">Demo</a><a href="/pricing" class="rounded-[8px] px-3 py-2 text-[14px] text-[#a1a1a1] transition-colors hover:bg-white/5 hover:text-white">Pricing</a><a href="/docs" class="rounded-[8px] px-3 py-2 text-[14px] text-[#a1a1a1] transition-colors hover:bg-white/5 hover:text-white">Docs</a><div class="group relative"><button type="button" class="inline-flex appearance-none items-center gap-1 rounded-[8px] border-0 bg-transparent px-3 py-2 text-[14px] font-normal text-[#ededed] transition-colors hover:bg-white/5 hover:text-white">Company<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" style="transform:none;transition:transform 180ms"><path d="m6 9 6 6 6-6"></path></svg></button><div class="absolute left-0 top-full w-[210px] pt-4 group-focus-within:block group-hover:block hidden"><div class="HomePage_megaMenuCompact__kKqk6 p-1.5"><div class="space-y-0.5"><a href="/blog" class="HomePage_menuItemCompact__reks1"><div class="flex items-center gap-3"><span class="inline-flex h-7 w-7 shrink-0 items-center justify-center rounded-[8px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-3.5 w-3.5" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M19.5 7.5v9.75A2.25 2.25 0 0117.25 19.5H6.75A2.25 2.25 0 014.5 17.25V6.75A2.25 2.25 0 016.75 4.5h10.5A2.25 2.25 0 0119.5 6.75V7.5Zm0 0H21v9.75A2.25 2.25 0 0118.75 19.5M8.25 8.25h7.5m-7.5 3h7.5m-7.5 3h4.5"></path></svg></span><span class="min-w-0 flex-1"><span class="block text-[14px] font-medium text-white">Blog</span></span></div></a><a href="/comparisons" class="HomePage_menuItemCompact__reks1"><div class="flex items-center gap-3"><span class="inline-flex h-7 w-7 shrink-0 items-center justify-center rounded-[8px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-3.5 w-3.5" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M12 7.5h1.5m-1.5 3h1.5m-7.5 3h7.5m-7.5 3h7.5m3-9h3.375c.621 0 1.125.504 1.125 1.125V18a2.25 2.25 0 01-2.25 2.25M16.5 7.5V18a2.25 2.25 0 002.25 2.25M16.5 7.5V4.875c0-.621-.504-1.125-1.125-1.125H4.125C3.504 3.75 3 4.254 3 4.875V18a2.25 2.25 0 002.25 2.25h13.5M6 7.5h3v3H6v-3Z"></path></svg></span><span class="min-w-0 flex-1"><span class="block text-[14px] font-medium text-white">Comparisons</span></span></div></a><a href="/knowledge-base" class="HomePage_menuItemCompact__reks1"><div class="flex items-center gap-3"><span class="inline-flex h-7 w-7 shrink-0 items-center justify-center rounded-[8px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-3.5 w-3.5" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5 5.754 5 4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18c1.746 0 3.332.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.746 0 3.332.477 4.5 1.253v13C19.832 18.477 18.246 18 16.5 18c-1.746 0-3.332.477-4.5 1.253"></path></svg></span><span class="min-w-0 flex-1"><span class="block text-[14px] font-medium text-white">Knowledge Base</span></span></div></a><a href="/careers" class="HomePage_menuItemCompact__reks1"><div class="flex items-center gap-3"><span class="inline-flex h-7 w-7 shrink-0 items-center justify-center rounded-[8px] border border-white/10 bg-white/[0.03] text-[#ededed]"><svg class="h-3.5 w-3.5" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M20.25 14.15v4.073c0 .414-.336.75-.75.75H4.5a.75.75 0 01-.75-.75V14.15m16.5 0V8.25A2.25 2.25 0 0018 6h-3.75V4.875A1.875 1.875 0 0012.375 3h-.75A1.875 1.875 0 009.75 4.875V6H6A2.25 2.25 0 003.75 8.25v5.9m16.5 0a48.108 48.108 0 01-16.5 0"></path></svg></span><span class="min-w-0 flex-1"><span class="block text-[14px] font-medium text-white">Careers</span></span></div></a></div></div></div></div></nav><div class="ml-auto hidden items-center gap-3 lg:flex"><a href="https://us-east.metoro.io" class="HomePage_buttonTertiary__tdko4">Log In<span class="HomePage_keycap__f6e4m">L</span></a><a href="https://us-east.metoro.io/register" class="HomePage_buttonPrimary__SIpyo">Sign up<span class="HomePage_keycapDark__CSdeE">S</span></a></div><button type="button" class="ml-auto inline-flex h-10 w-10 items-center justify-center rounded-[8px] border border-white/10 text-white transition-colors hover:bg-white/5 lg:hidden" aria-expanded="false" aria-controls="home-mobile-menu" aria-label="Open menu"><span class="relative block h-4 w-5"><span class="absolute left-0 top-0 h-[2px] w-5 bg-current transition-all "></span><span class="absolute left-0 top-[7px] h-[2px] w-5 bg-current transition-opacity opacity-100"></span><span class="absolute left-0 top-[14px] h-[2px] w-5 bg-current transition-all "></span></span></button></div></header></div><div class="article-wrapper article-dark"><div class="fixed top-0 left-0 right-0 h-[3px] z-[100] bg-transparent pointer-events-none"><div class="h-full bg-[var(--article-link)] transition-[width] duration-150 ease-out" style="width:0%;box-shadow:0 0 10px var(--article-link), 0 0 20px var(--article-link)"></div></div><div class="min-h-screen bg-[var(--article-bg-secondary)] relative transition-colors duration-300"><script type="application/ld+json">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Distributed Tracing: A Whistle Stop Tour","description":"Know enough to be dangerous in 10 minutes","keywords":"distributed tracing, tracing, observability, microservices, OpenTelemetry, Jaeger, Zipkin","mainEntityOfPage":{"@type":"WebPage","@id":"https://metoro.io/blog/distributed-tracing-whistle-stop-tour"},"publisher":{"@type":"Organization","name":"Metoro","logo":{"@type":"ImageObject","url":"https://metoro.io/static/images/Metoro_16_9.svg"}},"datePublished":"2024-03-13","dateModified":"2024-03-13","author":{"@type":"Person","name":"Chris Battarbee","url":"https://metoro.io/authors/chris-battarbee","description":"CEO, ex-Palantir Senior Software Engineer on Compute and Kubernetes"},"timeRequired":"PT11M"}</script><script type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://metoro.io/"},{"@type":"ListItem","position":2,"name":"Blog","item":"https://metoro.io/blog"},{"@type":"ListItem","position":3,"name":"Distributed Tracing: A Whistle Stop Tour","item":"https://metoro.io/blog/distributed-tracing-whistle-stop-tour"}]}</script><aside class="hidden lg:block fixed left-4 top-1/2 -translate-y-1/2 z-20"><div style="opacity:1;pointer-events:auto;transition:opacity 200ms ease-out"><button class="-ml-4 flex items-center justify-center px-2 py-4 rounded-r-lg border border-l-0 border-[var(--article-border)] bg-[var(--article-bg)] hover:bg-[var(--article-bg)] transition-colors font-sora text-[12px] text-[var(--article-text-secondary)] hover:text-[var(--article-text)]" aria-label="Open table of contents" style="writing-mode:vertical-rl">Contents</button></div></aside><div class="relative z-10 mx-auto max-w-4xl px-6 lg:px-8 pb-24 pt-8"><nav class="flex items-center text-sm text-[var(--article-text-secondary)] mb-8 transition-colors" aria-label="Breadcrumb"><a class="hover:text-[var(--article-heading)] transition-colors" href="/">Home</a><span class="mx-2 text-[var(--article-text-secondary)]">/</span><a class="hover:text-[var(--article-heading)] transition-colors" href="/blog">Blog</a><span><span class="mx-2 text-[var(--article-text-secondary)]">/</span><span class="text-[var(--article-heading)] capitalize transition-colors">distributed tracing whistle stop tour</span></span></nav><article><header class="mb-10"><div class="flex items-start justify-between gap-4 pr-4"><div class="flex-1"><h1 class="font-sora font-semibold text-[28px] sm:text-[32px] lg:text-[36px] tracking-[-0.5px] leading-[1.2] text-[var(--article-heading)] mb-4 transition-colors">Distributed Tracing: A Whistle Stop Tour</h1></div><button class="flex items-center gap-2 px-3 py-1.5 rounded-lg text-sm transition-colors bg-[var(--article-border)]/50 hover:bg-[var(--article-border)] text-[var(--article-text-secondary)] hover:text-[var(--article-text)]" aria-label="Current theme: System. Click to change." title="Theme: System"><svg class="w-5 h-5" fill="none" viewBox="0 0 24 24" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9.75 17L9 20l-1 1h8l-1-1-.75-3M3 13h18M5 17h14a2 2 0 002-2V5a2 2 0 00-2-2H5a2 2 0 00-2 2v10a2 2 0 002 2z"></path></svg><span class="hidden sm:inline">System</span></button></div><p class="font-sans font-normal text-[var(--article-text)] text-[16px] sm:text-[18px] leading-[1.6] mb-5 max-w-[800px] transition-colors">Know enough to be dangerous in 10 minutes</p><div class="flex items-center text-sm text-[var(--article-text-secondary)] space-x-4 flex-wrap font-sora transition-colors"><a href="/authors/chris-battarbee">By <span class="text-[var(--article-link)]">Chris Battarbee</span></a><div class="flex items-center space-x-1"><span>Published:</span><time dateTime="2024-03-13">March 13, 2024</time></div><div class="flex items-center space-x-1"><span>11<!-- --> min read</span></div></div></header><div class="jsx-a63b9aca72e45e85 knowledge-base-content"><p>If you're newer to the architecture side of this problem, pair this with <a href="/blog/microservice-interview-questions">microservice interview questions</a> before diving into tracing internals.</p>
<h2 id="what-problem-do-distributed-traces-solve">What problem do distributed traces solve?</h2>
<p>As the number of services in a distributed system increases, so does the difficulty of debugging the system when it fails or degrades. Issues stemming in one part of the system can impact services in an entirely different part of the system.</p>
<p>In the architecture below, we have a request passing through 3 micro-services and finally calling a database. If the database takes a long time to answer service C's query then the entire request would take a long time to return a response. The latency of each downstream service would increase dramatically.</p>
<img alt="Diagram showing request flow: Client → serviceA → B → C → database → (returns response) -> C → B → A → Client" src="/static/images/knowledge_base/flow-graph-1-distributed-tracing-whistle-stop-tour.png" style="background-color: #f8f9fa; padding: 10px; border-radius: 8px;">
<p>The team responsible for service A may have an alert configured to notify them if the service's latency increases, as high latency results in a poor user experience.</p>
<p>How would that engineer who owns service A start to debug this issue without distributed tracing?</p>
<p>The engineer likely knows that the call to service B is slow, but they might not understand how service B is built or what its dependencies are. They would need to consult with another engineer familiar with service B or examine service B's metrics (or code) to identify issues. This investigation would continue across all related services until they reach the database. Not exactly the most efficient process...</p>
<p>We need a method to track the journey of a request through a distributed system and identify the latency or errors at each stage.</p>
<p>This is the purpose of distributed tracing.</p>
<p>Using data emitted through distributed tracing instrumentation, we can understand the entire lifecycle of the request through all micro-services so any engineer can locate the source of the issue across the entire system quickly.</p>
<p>In this scenario, our engineer can examine a <a href="https://www.brendangregg.com/flamegraphs.html">flame graph</a> visualisation of a slow trace and quickly identify, as shown in the diagram below, that the database was responsible for the increased latency.</p>
<img alt="Flame graph with stacked bars: Database, ServiceC, ServiceB, ServiceA, and a timeline arrow showing call duration" src="/static/images/knowledge_base/stacked-bars-traces.png" style="background-color: #f8f9fa; padding: 10px; border-radius: 8px;">
<h2 id="how-do-distributed-traces-work">How do distributed traces work?</h2>
<p>Before understanding how traces are created and used. We first have to understand what a trace is.</p>
<div style="background-color: #1a1f2e; border-left: 4px solid #3793ff; padding: 16px 20px; margin: 24px 0; border-radius: 0 8px 8px 0;">
<div style="display: flex; align-items: flex-start; gap: 12px;">
<span style="font-size: 20px;">💡</span>
<div>
<strong style="color: #ffffff; font-weight: 600;">NOTE</strong>
<p style="color: #cecfd2; margin-top: 8px; margin-bottom: 0; line-height: 1.6;">
There are several projects which have different data models to represent a trace, the most notable ones are <a href="https://github.com/openzipkin/zipkin-api/blob/master/thrift/zipkinCore.thrift" style="color: #77edb0;">Zipkin</a>, <a href="https://www.jaegertracing.io/docs/1.23/architecture/#span" style="color: #77edb0;">Jaeger</a> and <a href="https://opentelemetry.io/docs/concepts/signals/traces/" style="color: #77edb0;">OpenTelemetry</a>. Despite some differences, their rough structure is the same. The industry seems to be settling on OpenTelemetry as the standard and so we'll be focusing on OpenTelemetry for the rest of this post.
</p>
</div>
</div>
</div>
<h3 id="what-is-a-trace">What is a trace?</h3>
<p>A trace consists of many <strong>spans</strong> in parent-child relationships. A trace is just a tree of spans.</p>
<p>Each <strong>span</strong> represents a unit of work and includes the following information (along with a few other details we won't cover here, as they are less important):</p>
<ul>
<li>
<p><strong>Span Name</strong> - The name of the operation being performed in the span e.g. <code>HTTP POST</code></p>
</li>
<li>
<p><strong>Parent Span Id</strong> - The span id of the parent span so that the tree can be reconstructed</p>
</li>
<li>
<p><strong>Start + End Timestamps</strong> - Used to calculate the overall time for the span to complete</p>
</li>
<li>
<p><strong>TraceId</strong> - The overarching identifier that the span is a part of, used to tie together all the spans together</p>
</li>
<li>
<p><strong>Attributes</strong> - Arbitrary key-value pairs used to store metadata about the span. Standardised naming patterns for keys have been <a href="https://pkg.go.dev/go.opentelemetry.io/otel@v1.24.0/semconv/v1.24.0">defined</a> to ensure information is consistently labeled across libraries. For example, if the operation involves an HTTP request, we can expect the method to be stored under the key <code>http.request.method</code>.</p>
</li>
<li>
<p><strong>Events</strong> - A list of meaningful things that happen at a specific point in time. E.g. <code>Page finished loading</code> at <code>2024-03-13T18:52:58.114561Z</code></p>
</li>
<li>
<p><strong>Links</strong> -A list of other span IDs (potentially in other traces) that are causally related to this span. This is useful for connecting operations that are asynchronous. For example, if someone buys an item from our online store, we might want to send them an email later. The email sending is triggered by the purchase request trace but isn't part of the same request, so we can connect them using a link.</p>
</li>
</ul>
<p>You might be thinking, how do I know when to start a new span?</p>
<p>There's no strict rule for the level of detail that would warrant starting a new span. You might have a single span for all the work done in one endpoint, or a span for each function call in your service. In general, the higher the level of detail, the more targeted and useful your traces will be, but a higher level of detail often comes with a larger upfront and ongoing maintenance cost. Different people handle this in various ways, and it often depends on how your service is instrumented. More on this topic later.</p>
<p>And that's pretty much it. A full trace looks something like this.</p>
<img alt="Trace X diagram with Span A as root, branching to Span B → Span D and Span C → Span E → Span G, plus Span F." src="/static/images/knowledge_base/traceX-with-spans.png" style="background-color: #f8f9fa; padding: 10px; border-radius: 8px;">
<h3 id="how-are-traces-generated">How are traces generated?</h3>
<p>As we now know, traces are made up of a bunch of related spans. So how are these spans generated?</p>
<p>Let's begin with a simple example: a trace that consists of a single span without any children.</p>
<p>To create a trace with a single span, we need to generate some identifiers, name the operation we're recording, track the duration, and then emit the span (we'll discuss where it goes later). In pseudo Go code, it might look like this.</p>
<pre class="language-go"><code class="language-go code-highlight"><span class="code-line"><span class="token comment">// Generate identifiers</span>
</span><span class="code-line">traceID <span class="token operator">:=</span> <span class="token function">generateTraceID</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">spanID <span class="token operator">:=</span> <span class="token function">generateSpanID</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">spanName <span class="token operator">:=</span> <span class="token string">"Important Operation To Record"</span>
</span><span class="code-line">
</span><span class="code-line">startTime <span class="token operator">:=</span> time<span class="token punctuation">.</span><span class="token function">Now</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token comment">// Perform the operation that we want to trace</span>
</span><span class="code-line"><span class="token function">doOperation</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line">endTime <span class="token operator">:=</span> time<span class="token punctuation">.</span><span class="token function">Now</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token function">emitSpan</span><span class="token punctuation">(</span>Span<span class="token punctuation">{</span>
</span><span class="code-line"> TraceId<span class="token punctuation">:</span> traceId<span class="token punctuation">,</span>
</span><span class="code-line"> ParentSpanId<span class="token punctuation">:</span> <span class="token boolean">nil</span><span class="token punctuation">,</span>
</span><span class="code-line"> SpanName<span class="token punctuation">:</span> spanName<span class="token punctuation">,</span>
</span><span class="code-line"> SpanId<span class="token punctuation">:</span> spanId<span class="token punctuation">,</span>
</span><span class="code-line"> StartTime<span class="token punctuation">:</span> startTime<span class="token punctuation">,</span>
</span><span class="code-line"> EndTime<span class="token punctuation">:</span> endTime
</span><span class="code-line"><span class="token punctuation">}</span><span class="token punctuation">)</span>
</span></code></pre>
<p>Simple enough right?</p>
<p>But now we want to add child spans to give us a more granular view of what's happening in our service. To emit a child span, we need to know two things:</p>
<ol>
<li>
<p>The traceId generated when we create the first span for the trace</p>
</li>
<li>
<p>The spanId of the parent span</p>
</li>
</ol>
<p>Okay, so now we need to pass around these two identifiers and we should be good to go. Let's pass around the traceId and current span id around in a <a href="https://go.dev/blog/context">context</a>. Now our pseudo span generation code looks something like this.</p>
<pre class="language-go"><code class="language-go code-highlight"><span class="code-line"><span class="token comment">// Get the traceId if it exists, otherwise generate one</span>
</span><span class="code-line">traceId <span class="token operator">:=</span> ctx<span class="token punctuation">.</span><span class="token function">Value</span><span class="token punctuation">(</span><span class="token string">"traceId"</span><span class="token punctuation">)</span>
</span><span class="code-line"><span class="token keyword">if</span> traceId <span class="token operator">==</span> <span class="token boolean">nil</span> <span class="token punctuation">{</span>
</span><span class="code-line"> traceId <span class="token operator">=</span> <span class="token function">generateTraceID</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line"><span class="token punctuation">}</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token comment">// Get the parent spanId if it exists</span>
</span><span class="code-line">parentSpanId <span class="token operator">:=</span> ctx<span class="token punctuation">.</span><span class="token function">Value</span><span class="token punctuation">(</span><span class="token string">"spanId"</span><span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token comment">// Generate the new child spanId</span>
</span><span class="code-line">spanId <span class="token operator">:=</span> <span class="token function">generateSpanID</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">spanName <span class="token operator">:=</span> <span class="token string">"Important Operation To Record"</span>
</span><span class="code-line">
</span><span class="code-line">startTime <span class="token operator">:=</span> time<span class="token punctuation">.</span><span class="token function">Now</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token comment">// Perform the operation that we want to trace</span>
</span><span class="code-line"><span class="token comment">// Pass down the context with the new spanId incase we want to make a new child span where this span is the parent</span>
</span><span class="code-line">ctxWithParentSpanId <span class="token operator">:=</span> context<span class="token punctuation">.</span><span class="token function">WithValue</span><span class="token punctuation">(</span>ctx<span class="token punctuation">,</span> <span class="token string">"spanId"</span><span class="token punctuation">,</span> spanId<span class="token punctuation">)</span>
</span><span class="code-line"><span class="token function">doOperation</span><span class="token punctuation">(</span>ctxWithParentSpanId<span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line">endTime <span class="token operator">:=</span> time<span class="token punctuation">.</span><span class="token function">Now</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token function">emitSpan</span><span class="token punctuation">(</span>Span<span class="token punctuation">{</span>
</span><span class="code-line"> TraceId<span class="token punctuation">:</span> traceId<span class="token punctuation">,</span>
</span><span class="code-line"> ParentSpanId<span class="token punctuation">:</span> parentSpanId<span class="token punctuation">,</span>
</span><span class="code-line"> SpanName<span class="token punctuation">:</span> spanName<span class="token punctuation">,</span>
</span><span class="code-line"> SpanId<span class="token punctuation">:</span> spanId<span class="token punctuation">,</span>
</span><span class="code-line"> StartTime<span class="token punctuation">:</span> startTime<span class="token punctuation">,</span>
</span><span class="code-line"> EndTime<span class="token punctuation">:</span> endTime
</span><span class="code-line"><span class="token punctuation">}</span><span class="token punctuation">)</span>
</span></code></pre>
<p>This works well enough until we need to pass values across the network so that we can create child spans in different services.</p>
<p>Different protocols have different ways of doing this. For HTTP, <a href="https://www.w3.org/TR/trace-context/#trace-context-http-headers-format">W3C recommends</a> that this information is passed around in the <code>traceparent</code> header. It's then up to the service on the other side of the request to pull this information out of http request and propagate it down through all the code handling the request. Giving us something like this for a system with 3 microservices that communicate via HTTP.</p>
<img alt="Sequence diagram of a client request traced through Service A, Service B, and Service C, with each generating child spans and returning the response to the client." src="/static/images/knowledge_base/detailed-flow-graph-distributed-tracing.png" style="background-color: #f8f9fa; padding: 10px; border-radius: 8px;">
<p>So now we need some code which adds the <code>traceparent</code> header to our http requests. It might look like something like this for a request made with the standard go http client</p>
<pre class="language-go"><code class="language-go code-highlight"><span class="code-line"><span class="token comment">// Get the traceId if it exists, otherwise generate one</span>
</span><span class="code-line">traceId <span class="token operator">:=</span> ctx<span class="token punctuation">.</span><span class="token function">Value</span><span class="token punctuation">(</span><span class="token string">"traceId"</span><span class="token punctuation">)</span>
</span><span class="code-line"><span class="token keyword">if</span> traceId <span class="token operator">==</span> <span class="token boolean">nil</span> <span class="token punctuation">{</span>
</span><span class="code-line"> traceId <span class="token operator">=</span> <span class="token function">generateTraceID</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line"><span class="token punctuation">}</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token comment">// Generate the new child spanId</span>
</span><span class="code-line">spanId <span class="token operator">:=</span> <span class="token function">generateSpanID</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token comment">// Get the parent spanId if it exists</span>
</span><span class="code-line">parentSpanId <span class="token operator">:=</span> ctx<span class="token punctuation">.</span><span class="token function">Value</span><span class="token punctuation">(</span><span class="token string">"spanId"</span><span class="token punctuation">)</span>
</span><span class="code-line">spanName <span class="token operator">:=</span> <span class="token string">"HTTP Operation"</span>
</span><span class="code-line">
</span><span class="code-line">startTime <span class="token operator">:=</span> time<span class="token punctuation">.</span><span class="token function">Now</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token comment">// Create a new context with the spanId, indicating this span is now the parent for any child spans</span>
</span><span class="code-line">ctxWithSpanId <span class="token operator">:=</span> context<span class="token punctuation">.</span><span class="token function">WithValue</span><span class="token punctuation">(</span>ctx<span class="token punctuation">,</span> <span class="token string">"spanId"</span><span class="token punctuation">,</span> spanId<span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token comment">// Create an HTTP client and request</span>
</span><span class="code-line">httpClient <span class="token operator">:=</span> <span class="token function">createHttpClient</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">httpRequest <span class="token operator">:=</span> <span class="token function">createHttpRequest</span><span class="token punctuation">(</span><span class="token string">"http://my-upstream-microservice/api/endpoint"</span><span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token comment">// Add the traceparent header to the HTTP request</span>
</span><span class="code-line"><span class="token comment">// The traceparent header format: traceId-parentSpanId (simplified)</span>
</span><span class="code-line">traceparentValue <span class="token operator">:=</span> fmt<span class="token punctuation">.</span><span class="token function">Sprintf</span><span class="token punctuation">(</span><span class="token string">"%s-%s"</span><span class="token punctuation">,</span> traceId<span class="token punctuation">,</span> spanId<span class="token punctuation">)</span>
</span><span class="code-line">httpRequest<span class="token punctuation">.</span>Header<span class="token punctuation">.</span><span class="token function">Add</span><span class="token punctuation">(</span><span class="token string">"traceparent"</span><span class="token punctuation">,</span> traceparentValue<span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"><span class="token comment">// Perform the HTTP call</span>
</span><span class="code-line">response <span class="token operator">:=</span> httpClient<span class="token punctuation">.</span><span class="token function">Do</span><span class="token punctuation">(</span>httpRequest<span class="token punctuation">,</span> ctxWithSpanId<span class="token punctuation">)</span>
</span></code></pre>
<p>We also need code to read the value of the HTTP header when we receive a request in our chosen web framework. This is how it might look using Go's standard library HTTP server.</p>
<pre class="language-go"><code class="language-go code-highlight"><span class="code-line"><span class="token keyword">func</span> <span class="token function">handler</span><span class="token punctuation">(</span>w http<span class="token punctuation">.</span>ResponseWriter<span class="token punctuation">,</span> r <span class="token operator">*</span>http<span class="token punctuation">.</span>Request<span class="token punctuation">)</span> <span class="token punctuation">{</span>
</span><span class="code-line"> <span class="token comment">// Extract the traceparent header</span>
</span><span class="code-line"> traceparentHeader <span class="token operator">:=</span> r<span class="token punctuation">.</span>Header<span class="token punctuation">.</span><span class="token function">Get</span><span class="token punctuation">(</span><span class="token string">"traceparent"</span><span class="token punctuation">)</span>
</span><span class="code-line">
</span><span class="code-line"> <span class="token comment">// Parse the traceparent header</span>
</span><span class="code-line"> parts <span class="token operator">:=</span> strings<span class="token punctuation">.</span><span class="token function">Split</span><span class="token punctuation">(</span>traceparentHeader<span class="token punctuation">,</span> <span class="token string">"-"</span><span class="token punctuation">)</span>
</span><span class="code-line"> traceId <span class="token operator">:=</span> parts<span class="token punctuation">[</span><span class="token number">0</span><span class="token punctuation">]</span>
</span><span class="code-line"> parentSpanId <span class="token operator">:=</span> parts<span class="token punctuation">[</span><span class="token number">1</span><span class="token punctuation">]</span>
</span><span class="code-line">
</span><span class="code-line"> <span class="token comment">// Pass these around to the rest of our code</span>
</span><span class="code-line"><span class="token punctuation">}</span>
</span></code></pre>
<p>Now writing all of this code to instrument communication libraries and frameworks and emitting traces seems like a lot of work.</p>
<p>Can we automate this? Why yes, we can.</p>
<h3 id="automated-distributed-tracing-instrumentation">Automated distributed tracing instrumentation</h3>
<p>There are three broad buckets of automated distributed tracing instrumentation. We'll dive deeper into each of these in a subsequent post, but for now here's a quick overview of each of them.</p>
<h4 id="compile-time-instrumentation">Compile Time Instrumentation</h4>
<p>This approach uses libraries or plugins added to existing libraries to incorporate instrumentation into the application. These plugins or libraries are included at compile time and require application level changes.</p>
<p>For example, the <a href="https://pkg.go.dev/go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin">otelgin</a> Go library offers middleware that automatically propagates trace metadata for all requests passing through the Gin HTTP server, similar to the process we coded by hand before.</p>
<pre class="language-go"><code class="language-go code-highlight"><span class="code-line">r <span class="token operator">:=</span> gin<span class="token punctuation">.</span><span class="token function">New</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
</span><span class="code-line">r<span class="token punctuation">.</span><span class="token function">Use</span><span class="token punctuation">(</span>otelgin<span class="token punctuation">.</span><span class="token function">Middleware</span><span class="token punctuation">(</span><span class="token string">"my_service_name"</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
</span><span class="code-line"><span class="token operator">...</span>
</span></code></pre>
<p>This approach is effective across a wide variety of languages and libraries, offering the most flexibility but also being the most time-consuming.</p>
<h4 id="automated-language-level-runtime-instrumentation">Automated Language Level Runtime Instrumentation</h4>
<p>This approach injects instrumentation into the application at runtime. This is a great option for languages which have a dynamic execution environment like python, java or node.js. An example of tooling targeting this class of instrumentation is the <a href="https://opentelemetry.io/docs/kubernetes/operator/automatic/">OpenTelemetry kubernetes operator</a> which injects instrumentation libraries into containers at startup depending on their language.</p>
<p>The injected code automatically instruments a number of popular libraries in the specified language so as long as you're using one of the supported libraries then this is a great option which tends to work better than the the kernel level instrumentation and also doesn't require an application level change.</p>
<h4 id="automated-kernel-runtime-instrumentation">Automated Kernel Runtime Instrumentation</h4>
<p>This instrumentation approach adds instrumentation at the level of the Kernel. In linux these tools tend to be based on <a href="https://ebpf.io/">eBPF</a> like <a href="https://cilium.io/">Cilium</a> or <a href="https://github.com/grafana/beyla">Beyla</a>.</p>
<p>They intercept <a href="https://man7.org/linux/man-pages/man2/syscalls.2.html">syscalls</a> and automatically emit spans. These tools don't require services (or their runtime) to be modified in any way and work with all languages so they're a great choice to get started with. However, the instrumentation is often coarse as it only has access to a small amount of information about the service.</p>
<p>Okay, now we know what a trace is, how spans link together and how we can get our applications instrumented and producing telemetry without too much manual work.</p>
<p>The next step is to get those spans exported somewhere they can be useful.</p>
<h3 id="how-are-traces-actually-emitted-and-collected">How are traces actually emitted and collected?</h3>
<p>In order for trace information to be useful, we need to have a centralised location where we can query them. This location could be a commercial cloud observability provider like <a href="https://www.datadoghq.com/">DataDog</a> or an internally hosted trace store like <a href="https://www.jaegertracing.io/">Jaeger</a> or <a href="https://github.com/grafana/tempo">Tempo</a>.</p>
<p>When emitting your trace data to a trace store you have a couple of choices.</p>
<ol>
<li>
<p>Ship the data to the store directly from the span creating service</p>
</li>
<li>
<p>Ship the data to a collector which then ships it to the store</p>
</li>
</ol>
<p>Generally, using an intermediate collector like the <a href="https://opentelemetry.io/docs/collector/">OpenTelemetry Collector</a> is a better choice as it gives you access to more advanced techniques like <a href="https://opentelemetry.io/blog/2022/tail-sampling/">tail sampling</a> and is more robust as your production services can offload the data quickly without worrying about retries, batching and a bunch of other concerns.</p>
<p>Each service establishes a connection with a collector and sends spans over the network as they're generated, open telemetry defines the <a href="https://opentelemetry.io/docs/specs/otlp/">OTLP</a> protocol for sending telemetry which sits on top of gRPC or HTTP.</p>
<img alt="Diagram showing Service A, B, and C sending traces via OTLP to an OTel Collector, which forwards them with TraceStoreProtocol to a Trace Store." src="/static/images/knowledge_base/otlp-collector-trace-store.png" style="background-color: #f8f9fa; padding: 10px; border-radius: 8px;">
<p>Now we have our services producing spans, they're sent to the collector which forwards them to the trace store. The trace store indexes these spans by the traceId and a bunch of other characteristics so you can run queries to find interesting behaviour in your systems.</p>
<p>Now you have a solid idea of what traces are, how they're generated and an idea of what they can be used for.</p>
<h3 id="future-posts">Future posts</h3>
<p>I'll be writing a new blog post each week on this space so subscribe to the newsletter if you found this interesting and let me know what topics you'd like to see!</p></div><section class="mt-16 flex flex-col gap-5 rounded-[16px] border border-[var(--article-border)] bg-[var(--article-bg)] p-6 transition-colors sm:flex-row sm:items-center"><img alt="Chris Battarbee avatar" loading="lazy" width="64" height="64" decoding="async" data-nimg="1" class="h-16 w-16 flex-shrink-0 rounded-full border border-[var(--article-border)] object-cover" style="color:transparent" srcSet="/_next/image?url=%2Fstatic%2Fimages%2Favatars%2Fchris-battarbee.jpg&amp;w=64&amp;q=75 1x, /_next/image?url=%2Fstatic%2Fimages%2Favatars%2Fchris-battarbee.jpg&amp;w=128&amp;q=75 2x" src="/_next/image?url=%2Fstatic%2Fimages%2Favatars%2Fchris-battarbee.jpg&amp;w=128&amp;q=75"/><div><div class="font-sora text-[12px] uppercase tracking-[0.14em] text-[var(--article-text-secondary)] transition-colors">Written by</div><a href="/authors/chris-battarbee" class="mt-1 inline-block font-sora text-[18px] font-semibold text-[var(--article-link)] transition-colors hover:text-[var(--article-link-hover)]">Chris Battarbee</a><p class="mt-2 max-w-[620px] font-sans text-[15px] leading-[1.6] text-[var(--article-text)] transition-colors">CEO, ex-Palantir Senior Software Engineer on Compute and Kubernetes</p></div></section><section class="mt-16 border-t border-[var(--article-border)] pt-10 transition-colors"><div class="mb-6 max-w-2xl"><h2 class="font-sora font-semibold text-[24px] sm:text-[28px] tracking-[-0.5px] text-[var(--article-heading)] transition-colors">Related reading</h2><p class="mt-2 font-sans text-[15px] leading-[1.6] text-[var(--article-text-secondary)] transition-colors">More Metoro articles that deepen the same topic from another angle.</p></div><div class="grid grid-cols-1 gap-4 md:grid-cols-3"><a class="group flex h-full flex-col rounded-[16px] border border-[var(--article-border)] bg-[var(--article-bg)] p-5 transition-colors hover:border-[var(--article-link)]" href="/blog/microservice-interview-questions"><time dateTime="2024-03-18" class="font-sora text-[11px] uppercase tracking-[0.18em] text-[var(--article-text-secondary)] transition-colors">Mar 18, 2024</time><h3 class="mt-3 font-sora text-[19px] font-semibold leading-[1.35] text-[var(--article-heading)] transition-colors group-hover:text-[var(--article-link)]">Microservice Interview Questions: The basics</h3><p class="mt-3 flex-1 font-sans text-[15px] leading-[1.6] text-[var(--article-text)] transition-colors">Detailed answers to common questions about microservices, starting from the basics and getting into the details</p><span class="mt-5 font-sora text-[14px] text-[var(--article-link)] transition-colors group-hover:text-[var(--article-link-hover)]">Read article →</span></a><a class="group flex h-full flex-col rounded-[16px] border border-[var(--article-border)] bg-[var(--article-bg)] p-5 transition-colors hover:border-[var(--article-link)]" href="/blog/how-metoro-uses-ebpf"><time dateTime="2026-01-27" class="font-sora text-[11px] uppercase tracking-[0.18em] text-[var(--article-text-secondary)] transition-colors">Jan 27, 2026</time><h3 class="mt-3 font-sora text-[19px] font-semibold leading-[1.35] text-[var(--article-heading)] transition-colors group-hover:text-[var(--article-link)]">How Metoro Uses eBPF for Zero-Instrumentation Observability</h3><p class="mt-3 flex-1 font-sans text-[15px] leading-[1.6] text-[var(--article-text)] transition-colors">A technical deep-dive into how Metoro captures L7 protocol traffic and intercepts TLS-encrypted data using eBPF, enabling automatic observability without code changes</p><span class="mt-5 font-sora text-[14px] text-[var(--article-link)] transition-colors group-hover:text-[var(--article-link-hover)]">Read article →</span></a><a class="group flex h-full flex-col rounded-[16px] border border-[var(--article-border)] bg-[var(--article-bg)] p-5 transition-colors hover:border-[var(--article-link)]" href="/blog/kubernetes-observability"><time dateTime="2026-02-03" class="font-sora text-[11px] uppercase tracking-[0.18em] text-[var(--article-text-secondary)] transition-colors">Feb 3, 2026</time><h3 class="mt-3 font-sora text-[19px] font-semibold leading-[1.35] text-[var(--article-heading)] transition-colors group-hover:text-[var(--article-link)]">Kubernetes Observability: The Complete Guide</h3><p class="mt-3 flex-1 font-sans text-[15px] leading-[1.6] text-[var(--article-text)] transition-colors">Learn what Kubernetes observability is and how to implement effective observability for your k8s clusters.</p><span class="mt-5 font-sora text-[14px] text-[var(--article-link)] transition-colors group-hover:text-[var(--article-link-hover)]">Read article →</span></a></div></section><div class="mt-16 pt-8 border-t border-[var(--article-border)] transition-colors"><div class="flex justify-between items-center"><a class="font-sora font-normal text-[var(--article-link)] text-[16px] hover:text-[var(--article-link-hover)] transition-colors flex items-center gap-2" href="/blog">← Back to <!-- -->Blog</a><a class="font-sora font-normal text-[16px] transition-colors flex items-center gap-2 text-[var(--article-link)] hover:text-[var(--article-link-hover)]" href="/docs">Read the Technical docs<!-- --> →</a></div></div></article></div></div><div class="bg-[#05080d]"><footer class="px-4 py-16 sm:px-6 lg:px-8 lg:py-20"><div class="mx-auto grid max-w-[1240px] gap-12 lg:grid-cols-[1.8fr_0.9fr_0.9fr_0.9fr_1.2fr]"><div><img alt="Metoro" loading="lazy" width="400" height="68" decoding="async" data-nimg="1" class="h-[40px] w-auto" style="color:transparent" src="/static/images/Metoro_16_9.svg"/><p class="mt-5 max-w-[360px] text-[11px] leading-[1.55] text-[#a1a1a1]">Metoro is an AI SRE and observability platform for teams running on Kubernetes. It automatically detects production issues, investigates alerts, verifies deployments, and finds root causes using built-in eBPF telemetry, Kubernetes context, and code-change analysis. Fast to install, available as Cloud, BYOC, or on-prem.</p><div class="mt-5 flex flex-wrap gap-2"><span class="HomePage_footerBadge__dIKUf">SOC 2 Type II</span><span class="HomePage_footerBadge__dIKUf">CNCF Silver</span><span class="HomePage_footerBadge__dIKUf">Linux Foundation</span></div></div><div><div class="HomePage_eyebrow__Gy9YW">Support</div><ul class="mt-4 space-y-3"><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/pricing">Pricing</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/docs">Documentation</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/docs">Guides</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/knowledge-base">Knowledge Base</a></li></ul></div><div><div class="HomePage_eyebrow__Gy9YW">Company</div><ul class="mt-4 space-y-3"><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/blog">Blog</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/comparisons">Comparisons</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/metoro-vs-datadog">Metoro vs Datadog</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/metoro-vs-grafana">Metoro vs Grafana</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/metoro-vs-dynatrace">Metoro vs Dynatrace</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/metoro-vs-newrelic">Metoro vs New Relic</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/metoro-vs-groundcover">Metoro vs Groundcover</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/metoro-vs-coroot">Metoro vs Coroot</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="https://www.ycombinator.com/companies/metoro">Y Combinator</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="mailto:jobs@metoro.io">Jobs</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="mailto:press@metoro.io">Press</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="mailto:partners@metoro.io">Partners</a></li></ul></div><div><div class="HomePage_eyebrow__Gy9YW">Legal</div><ul class="mt-4 space-y-3"><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="https://app.termly.io/policy-viewer/policy.html?policyUUID=576ce06b-a98c-494a-afd5-09556c84239b">Privacy</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="https://app.termly.io/policy-viewer/policy.html?policyUUID=d09c3e35-5dad-4fac-abf4-ad22a1f1d7d4">Terms</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="https://app.termly.io/policy-viewer/policy.html?policyUUID=cc4754b5-63af-4a13-b5b5-c86d45c1873b">Cookie Policy</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/sla">SLA</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/subprocessors">Subprocessors</a></li><li><a class="text-[13px] text-[#a1a1a1] transition-colors hover:text-white" href="/ai-usage">AI Usage</a></li></ul></div><div><div class="HomePage_eyebrow__Gy9YW">Subscribe</div><p class="HomePage_subscribeCopy__7lTwx mt-4">The latest news, articles, and resources, weekly.</p><div class="mt-4 flex flex-col gap-2 sm:flex-row"><label for="homepage-newsletter-email" class="sr-only">Email address</label><input id="homepage-newsletter-email" type="email" placeholder="Email address" class="min-w-0 flex-1 rounded-[8px] border border-[#262626] bg-[#111111] px-3 py-2 text-[13px] text-white outline-none transition-colors placeholder:text-[#707070] focus:border-[#34b1e6]" value=""/><button type="button" class="inline-flex items-center justify-center rounded-[8px] bg-white px-4 py-2 text-[13px] font-medium text-black transition-transform hover:-translate-y-[1px]">Subscribe</button></div></div></div><div class="HomePage_footerDivider__EGwz5 mx-auto mt-12 grid gap-4 pt-6 text-[12px] text-[#707070] md:grid-cols-[1fr_auto_1fr] md:items-center"><span class="block md:col-start-1 md:row-start-1">© <!-- -->2026<!-- --> Metoro, Inc. All rights reserved. SOC 2 Type II Certified.</span><div class="md:col-start-2 md:row-start-1 md:justify-self-center"><div class="flex items-center space-x-2"><div class="h-2 w-2 animate-pulse rounded-full bg-gray-500"></div><span class="text-xs text-gray-400">Loading status...</span></div></div><div class="md:col-start-3 md:row-start-1 md:justify-self-end"><span class="flex items-center gap-4"><a href="https://join.slack.com/t/metorocommunity/shared_invite/zt-2makpjl5j-F0WcpGnPcdc8anbNGcewqw" class="text-[#707070] transition-colors hover:text-white" aria-label="Slack"><svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 512 512" class="h-5 w-5"><path d="M126.12,315.1A47.06,47.06,0,1,1,79.06,268h47.06Z"></path><path d="M149.84,315.1a47.06,47.06,0,0,1,94.12,0V432.94a47.06,47.06,0,1,1-94.12,0Z"></path><path d="M196.9,126.12A47.06,47.06,0,1,1,244,79.06v47.06Z"></path><path d="M196.9,149.84a47.06,47.06,0,0,1,0,94.12H79.06a47.06,47.06,0,0,1,0-94.12Z"></path><path d="M385.88,196.9A47.06,47.06,0,1,1,432.94,244H385.88Z"></path><path d="M362.16,196.9a47.06,47.06,0,0,1-94.12,0V79.06a47.06,47.06,0,1,1,94.12,0Z"></path><path d="M315.1,385.88A47.06,47.06,0,1,1,268,432.94V385.88Z"></path><path d="M315.1,362.16a47.06,47.06,0,0,1,0-94.12H432.94a47.06,47.06,0,1,1,0,94.12Z"></path></svg></a><a href="https://x.com/metoro_ai" class="text-[#707070] transition-colors hover:text-white" aria-label="X"><svg fill="currentColor" viewBox="0 0 24 24" class="h-5 w-5"><path d="M13.6823 10.6218 20.2391 3H18.6854L12.9921 9.61788 8.44486 3H3.2002L10.0765 13.0074 3.2002 21H4.75404L10.7663 14.0113 15.5685 21H20.8131L13.6819 10.6218h.0004ZM11.5541 13.0956l-.6967-.9965-5.54349-7.92939H7.70053L12.1742 10.5689l.6967.9966 5.8152 8.318H16.2995L11.5541 13.096v-.0004Z"></path></svg></a><a href="https://github.com/metoro-io" class="text-[#707070] transition-colors hover:text-white" aria-label="GitHub"><svg fill="currentColor" viewBox="0 0 24 24" class="h-5 w-5"><path fill-rule="evenodd" d="M12 2C6.477 2 2 6.484 2 12.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0112 6.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.202 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.943.359.309.678.92.678 1.855 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0022 12.017C22 6.484 17.522 2 12 2Z" clip-rule="evenodd"></path></svg></a><a href="https://www.youtube.com/channel/UCecMYDdTv5Ch6DftPJvygiA" class="text-[#707070] transition-colors hover:text-white" aria-label="YouTube"><svg fill="currentColor" viewBox="0 0 24 24" class="h-5 w-5"><path fill-rule="evenodd" d="M19.812 5.418c.861.23 1.538.907 1.768 1.768C21.998 8.746 22 12 22 12s0 3.255-.418 4.814a2.504 2.504 0 0 1-1.768 1.768c-1.56.419-7.814.419-7.814.419s-6.255 0-7.814-.419a2.505 2.505 0 0 1-1.768-1.768C2 15.255 2 12 2 12s0-3.255.417-4.814a2.507 2.507 0 0 1 1.768-1.768C5.744 5 11.998 5 11.998 5s6.255 0 7.814.418ZM15.194 12 10 15V9l5.194 3Z" clip-rule="evenodd"></path></svg></a><a href="https://www.linkedin.com/company/98393849" class="text-[#707070] transition-colors hover:text-white" aria-label="LinkedIn"><svg fill="currentColor" viewBox="0 0 24 24" class="h-5 w-5"><path fill-rule="evenodd" d="M19 0h-14C2.239 0 0 2.239 0 5v14c0 2.761 2.239 5 5 5h14c2.762 0 5-2.239 5-5V5c0-2.762-2.238-5-5-5ZM8 19H5V8h3v11ZM6.5 6.732c-.966 0-1.75-.79-1.75-1.764S5.534 3.204 6.5 3.204s1.75.79 1.75 1.764S7.467 6.732 6.5 6.732ZM20 19h-3v-5.604c0-3.368-4-3.113-4 0V19h-3V8h3v1.765c1.396-2.586 7-2.777 7 2.476V19Z" clip-rule="evenodd"></path></svg></a></span></div></div></footer></div></div><!--$--><!--/$--></div><!--$--><!--/$--><script src="/_next/static/chunks/webpack-491b48b36c258e17.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/_next/static/media/1eedfae966de4e1c-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/_next/static/media/3e68fbb477d90635-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n3:HL[\"/_next/static/media/8935352d0bfcf3a9-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n4:HL[\"/_next/static/media/9f5251eb979d3d3e-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n5:HL[\"/_next/static/media/a325990d5fec1d3f-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n6:HL[\"/_next/static/media/a8803dd9fdd15a53-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n7:HL[\"/_next/static/media/e244d3ca994d3054-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n8:HL[\"/_next/static/media/e4af272ccee01ff0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n9:HL[\"/_next/static/media/f3f7e95f2dbc4fe4-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\na:HL[\"/_next/static/css/ba854656cbf121ba.css\",\"style\"]\nb:HL[\"/_next/static/css/960e6a5fafc45966.css\",\"style\"]\nc:HL[\"/_next/static/css/65af9316bbe810e3.css\",\"style\"]\nd:HL[\"/_next/static/css/bfde9e8e18a2a61f.css\",\"style\"]\ne:HL[\"/_next/static/css/7c28a0e1df995273.css\",\"style\"]\nf:HL[\"/_next/static/css/b5291668b3a6ced6.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"10:I[69820,[],\"\"]\n13:I[15309,[],\"\"]\n15:I[90676,[],\"\"]\n16:I[97926,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"\"]\n17:I[36844,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"InternalIntercomProvider\"]\n18:I[36844,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"PHProvider\"]\n19:I[36844,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6"])</script><script>self.__next_f.push([1,"159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"Providers\"]\n1a:I[67651,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"CalendlyProvider\"]\n1b:I[2048,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"default\"]\n1c:I[9302,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b"])</script><script>self.__next_f.push([1,"595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"HomeHeader\"]\n1d:I[32915,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"GoogleAds\"]\n1e:I[41500,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"MarketingAnalytics\"]\n1f:I[79536,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751"])</script><script>self.__next_f.push([1,"-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"MicrosoftAds\"]\n20:I[34029,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"SpeedInsights\"]\n21:I[72515,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"CookieConsentBanner\"]\n22:I[29189,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fbd51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"CommandPalette\"]\n23:I[97655,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6196\",\"static/chunks/fb"])</script><script>self.__next_f.push([1,"d51a90-4c969d592a1e29cb.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"6818\",\"static/chunks/6818-b595c894fced36f8.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"3185\",\"static/chunks/app/layout-46f53436efc58821.js\"],\"Analytics\"]\n25:I[61983,[],\"\"]\n14:[\"slug\",\"distributed-tracing-whistle-stop-tour\",\"c\"]\n26:[]\n"])</script><script>self.__next_f.push([1,"0:[\"$\",\"$L10\",null,{\"buildId\":\"-UvY4qZAzXXU8Yi95MSzK\",\"assetPrefix\":\"\",\"urlParts\":[\"\",\"blog\",\"distributed-tracing-whistle-stop-tour\"],\"initialTree\":[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"distributed-tracing-whistle-stop-tour\",\"c\"],{\"children\":[\"__PAGE__\",{}]}]}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"distributed-tracing-whistle-stop-tour\",\"c\"],{\"children\":[\"__PAGE__\",{},[[\"$L11\",\"$L12\",[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/b5291668b3a6ced6.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}]]],null],null]},[null,[\"$\",\"$L13\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"blog\",\"children\",\"$14\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L15\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\"}]],null]},[null,[\"$\",\"$L13\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"blog\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L15\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\"}]],null]},[[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/ba854656cbf121ba.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/960e6a5fafc45966.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}],[\"$\",\"link\",\"2\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/65af9316bbe810e3.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}],[\"$\",\"link\",\"3\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/bfde9e8e18a2a61f.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}],[\"$\",\"link\",\"4\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/7c28a0e1df995273.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"className\":\"h-full __variable_f367f3 __variable_b436a8 __variable_ce197d __variable_cfa357\",\"suppressHydrationWarning\":true,\"children\":[[\"$\",\"head\",null,{\"children\":[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"{\\\"@context\\\":\\\"https://schema.org\\\",\\\"@type\\\":\\\"WebSite\\\",\\\"name\\\":\\\"Metoro AI SRE\\\",\\\"url\\\":\\\"https://metoro.io\\\",\\\"publisher\\\":{\\\"@type\\\":\\\"Organization\\\",\\\"name\\\":\\\"Metoro AI SRE\\\",\\\"logo\\\":{\\\"@type\\\":\\\"ImageObject\\\",\\\"url\\\":\\\"https://metoro.io/static/images/Metoro_16_9.svg\\\"}}}\"}}]}],[\"$\",\"body\",null,{\"className\":\"flex min-h-full bg-[#05080d]\",\"children\":[[\"$\",\"$L16\",null,{\"id\":\"scroll-restoration\",\"strategy\":\"beforeInteractive\",\"dangerouslySetInnerHTML\":{\"__html\":\"\\n if ('scrollRestoration' in history) {\\n history.scrollRestoration = 'manual';\\n }\\n window.scrollTo(0, 0);\\n \"}}],[\"$\",\"$L17\",null,{\"appId\":\"uhk91085\",\"children\":[\"$\",\"$L18\",null,{\"children\":[\"$\",\"$L19\",null,{\"children\":[\"$\",\"$L1a\",null,{\"calendlyUrl\":\"https://calendly.com/chris-battarbee/metoro-engineer-intro\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex w-full flex-col\",\"children\":[[\"$\",\"$L1b\",null,{}],false,[\"$\",\"$L1c\",null,{}],[\"$\",\"$L1d\",null,{}],[\"$\",\"$L1e\",null,{}],[\"$\",\"$L1f\",null,{}],false,[\"$\",\"$L13\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L15\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[\"$\",\"div\",null,{\"className\":\"flex min-h-screen w-full bg-[#05080d]\",\"children\":[\"$\",\"div\",null,{\"className\":\"mx-auto flex min-w-0 w-full max-w-2xl flex-1 items-center justify-center px-4 pb-16 pt-36 lg:pl-8 lg:pr-0 sm:pt-44 xl:px-16\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex flex-col items-center justify-center text-center\",\"children\":[[\"$\",\"p\",null,{\"className\":\"font-display text-sm font-medium text-textlight\",\"children\":\"404\"}],[\"$\",\"h1\",null,{\"className\":\"mt-3 font-display text-3xl tracking-tight text-textlight\",\"children\":\"Page not found\"}],[\"$\",\"p\",null,{\"className\":\"mt-2 text-sm text-textmedium\",\"children\":\"Sorry, we couldn’t find the page you’re looking for.\"}],[\"$\",\"a\",null,{\"href\":\"/\",\"className\":\"mt-8 rounded-lg bg-primary px-4 py-2 text-sm font-semibold leading-6 text-white hover:bg-primary/80\",\"children\":\"Go back home\"}]]}]}]}],\"notFoundStyles\":[]}],[\"$\",\"$L20\",null,{}],[\"$\",\"$L21\",null,{}]]}],[\"$\",\"$L22\",null,{}]]}]}]}]}],[\"$\",\"$L23\",null,{}]]}]]}]],null],null],\"couldBeIntercepted\":false,\"initialHead\":[null,\"$L24\"],\"globalErrorComponent\":\"$25\",\"missingSlots\":\"$W26\"}]\n"])</script><script>self.__next_f.push([1,"27:I[64527,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"9587\",\"static/chunks/9587-30e87c7f19bde82a.js\",\"4136\",\"static/chunks/4136-66f49c1e9aae6bbc.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"5319\",\"static/chunks/5319-4ce901258ca1d364.js\",\"6797\",\"static/chunks/app/blog/%5B...slug%5D/page-8cfd0cc158c6d2e7.js\"],\"ArticleThemeWrapper\"]\n28:I[93910,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"9587\",\"static/chunks/9587-30e87c7f19bde82a.js\",\"4136\",\"static/chunks/4136-66f49c1e9aae6bbc.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"5319\",\"static/chunks/5319-4ce901258ca1d364.js\",\"6797\",\"static/chunks/app/blog/%5B...slug%5D/page-8cfd0cc158c6d2e7.js\"],\"default\"]\n29:I[49479,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"9587\",\"static/chunks/9587-30e87c7f19bde82a.js\",\"4136\",\"static/chunks/4136-66f49c1e9aae6bbc.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"5319\",\"static/chunks/5319-4ce901258ca1d364.js\",\"6797\",\"static/chunks/app/blog/%5B...slug%5D/page-8cfd0cc158c6d2e7.js\"],\"\"]\n2a:I[64527,[\"2832\""])</script><script>self.__next_f.push([1,",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"9587\",\"static/chunks/9587-30e87c7f19bde82a.js\",\"4136\",\"static/chunks/4136-66f49c1e9aae6bbc.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"5319\",\"static/chunks/5319-4ce901258ca1d364.js\",\"6797\",\"static/chunks/app/blog/%5B...slug%5D/page-8cfd0cc158c6d2e7.js\"],\"ArticleHeaderWithToggle\"]\n2b:I[15451,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"9587\",\"static/chunks/9587-30e87c7f19bde82a.js\",\"4136\",\"static/chunks/4136-66f49c1e9aae6bbc.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"5319\",\"static/chunks/5319-4ce901258ca1d364.js\",\"6797\",\"static/chunks/app/blog/%5B...slug%5D/page-8cfd0cc158c6d2e7.js\"],\"default\"]\n2d:I[84640,[\"2832\",\"static/chunks/d9202a3a-6c4efff19547c375.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"9587\",\"static/chunks/9587-30e87c7f19bde82a.js\",\"4136\",\"static/chunks/4136-66f49c1e9aae6bbc.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"5319\",\"static/chunks/5319-4ce901258ca1d364.js\",\"6797\",\"static/chunks/app/blog/%5B...slug%5D/page-8cfd0cc158c6d2e7.js\"],\"Image\"]\n2e:I[81740,[\"2832\",\"static/"])</script><script>self.__next_f.push([1,"chunks/d9202a3a-6c4efff19547c375.js\",\"6159\",\"static/chunks/6159-7d1f9e865c002a7f.js\",\"2707\",\"static/chunks/2707-8248e5f0f3ae475e.js\",\"7918\",\"static/chunks/7918-35abfc54082c840d.js\",\"9479\",\"static/chunks/9479-bbef253cea568311.js\",\"9587\",\"static/chunks/9587-30e87c7f19bde82a.js\",\"4136\",\"static/chunks/4136-66f49c1e9aae6bbc.js\",\"1740\",\"static/chunks/1740-64df73f896fd8888.js\",\"7651\",\"static/chunks/7651-9c1698f45dc2ca4f.js\",\"3310\",\"static/chunks/3310-8b48b40a4aa05b46.js\",\"2751\",\"static/chunks/2751-e2017637308c8f9c.js\",\"5319\",\"static/chunks/5319-4ce901258ca1d364.js\",\"6797\",\"static/chunks/app/blog/%5B...slug%5D/page-8cfd0cc158c6d2e7.js\"],\"HomeFooter\"]\n2c:T73fc,"])</script><script>self.__next_f.push([1,"\u003cp\u003eIf you're newer to the architecture side of this problem, pair this with \u003ca href=\"/blog/microservice-interview-questions\"\u003emicroservice interview questions\u003c/a\u003e before diving into tracing internals.\u003c/p\u003e\n\u003ch2 id=\"what-problem-do-distributed-traces-solve\"\u003eWhat problem do distributed traces solve?\u003c/h2\u003e\n\u003cp\u003eAs the number of services in a distributed system increases, so does the difficulty of debugging the system when it fails or degrades. Issues stemming in one part of the system can impact services in an entirely different part of the system.\u003c/p\u003e\n\u003cp\u003eIn the architecture below, we have a request passing through 3 micro-services and finally calling a database. If the database takes a long time to answer service C's query then the entire request would take a long time to return a response. The latency of each downstream service would increase dramatically.\u003c/p\u003e\n\u003cimg alt=\"Diagram showing request flow: Client → serviceA → B → C → database → (returns response) -\u003e C → B → A → Client\" src=\"/static/images/knowledge_base/flow-graph-1-distributed-tracing-whistle-stop-tour.png\" style=\"background-color: #f8f9fa; padding: 10px; border-radius: 8px;\"\u003e\n\u003cp\u003eThe team responsible for service A may have an alert configured to notify them if the service's latency increases, as high latency results in a poor user experience.\u003c/p\u003e\n\u003cp\u003eHow would that engineer who owns service A start to debug this issue without distributed tracing?\u003c/p\u003e\n\u003cp\u003eThe engineer likely knows that the call to service B is slow, but they might not understand how service B is built or what its dependencies are. They would need to consult with another engineer familiar with service B or examine service B's metrics (or code) to identify issues. This investigation would continue across all related services until they reach the database. Not exactly the most efficient process...\u003c/p\u003e\n\u003cp\u003eWe need a method to track the journey of a request through a distributed system and identify the latency or errors at each stage.\u003c/p\u003e\n\u003cp\u003eThis is the purpose of distributed tracing.\u003c/p\u003e\n\u003cp\u003eUsing data emitted through distributed tracing instrumentation, we can understand the entire lifecycle of the request through all micro-services so any engineer can locate the source of the issue across the entire system quickly.\u003c/p\u003e\n\u003cp\u003eIn this scenario, our engineer can examine a \u003ca href=\"https://www.brendangregg.com/flamegraphs.html\"\u003eflame graph\u003c/a\u003e visualisation of a slow trace and quickly identify, as shown in the diagram below, that the database was responsible for the increased latency.\u003c/p\u003e\n\u003cimg alt=\"Flame graph with stacked bars: Database, ServiceC, ServiceB, ServiceA, and a timeline arrow showing call duration\" src=\"/static/images/knowledge_base/stacked-bars-traces.png\" style=\"background-color: #f8f9fa; padding: 10px; border-radius: 8px;\"\u003e\n\u003ch2 id=\"how-do-distributed-traces-work\"\u003eHow do distributed traces work?\u003c/h2\u003e\n\u003cp\u003eBefore understanding how traces are created and used. We first have to understand what a trace is.\u003c/p\u003e\n\u003cdiv style=\"background-color: #1a1f2e; border-left: 4px solid #3793ff; padding: 16px 20px; margin: 24px 0; border-radius: 0 8px 8px 0;\"\u003e\n\u003cdiv style=\"display: flex; align-items: flex-start; gap: 12px;\"\u003e\n\u003cspan style=\"font-size: 20px;\"\u003e💡\u003c/span\u003e\n\u003cdiv\u003e\n\u003cstrong style=\"color: #ffffff; font-weight: 600;\"\u003eNOTE\u003c/strong\u003e\n\u003cp style=\"color: #cecfd2; margin-top: 8px; margin-bottom: 0; line-height: 1.6;\"\u003e\nThere are several projects which have different data models to represent a trace, the most notable ones are \u003ca href=\"https://github.com/openzipkin/zipkin-api/blob/master/thrift/zipkinCore.thrift\" style=\"color: #77edb0;\"\u003eZipkin\u003c/a\u003e, \u003ca href=\"https://www.jaegertracing.io/docs/1.23/architecture/#span\" style=\"color: #77edb0;\"\u003eJaeger\u003c/a\u003e and \u003ca href=\"https://opentelemetry.io/docs/concepts/signals/traces/\" style=\"color: #77edb0;\"\u003eOpenTelemetry\u003c/a\u003e. Despite some differences, their rough structure is the same. The industry seems to be settling on OpenTelemetry as the standard and so we'll be focusing on OpenTelemetry for the rest of this post.\n\u003c/p\u003e\n\u003c/div\u003e\n\u003c/div\u003e\n\u003c/div\u003e\n\u003ch3 id=\"what-is-a-trace\"\u003eWhat is a trace?\u003c/h3\u003e\n\u003cp\u003eA trace consists of many \u003cstrong\u003espans\u003c/strong\u003e in parent-child relationships. A trace is just a tree of spans.\u003c/p\u003e\n\u003cp\u003eEach \u003cstrong\u003espan\u003c/strong\u003e represents a unit of work and includes the following information (along with a few other details we won't cover here, as they are less important):\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\n\u003cp\u003e\u003cstrong\u003eSpan Name\u003c/strong\u003e - The name of the operation being performed in the span e.g. \u003ccode\u003eHTTP POST\u003c/code\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp\u003e\u003cstrong\u003eParent Span Id\u003c/strong\u003e - The span id of the parent span so that the tree can be reconstructed\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp\u003e\u003cstrong\u003eStart + End Timestamps\u003c/strong\u003e - Used to calculate the overall time for the span to complete\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp\u003e\u003cstrong\u003eTraceId\u003c/strong\u003e - The overarching identifier that the span is a part of, used to tie together all the spans together\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp\u003e\u003cstrong\u003eAttributes\u003c/strong\u003e - Arbitrary key-value pairs used to store metadata about the span. Standardised naming patterns for keys have been \u003ca href=\"https://pkg.go.dev/go.opentelemetry.io/otel@v1.24.0/semconv/v1.24.0\"\u003edefined\u003c/a\u003e to ensure information is consistently labeled across libraries. For example, if the operation involves an HTTP request, we can expect the method to be stored under the key \u003ccode\u003ehttp.request.method\u003c/code\u003e.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp\u003e\u003cstrong\u003eEvents\u003c/strong\u003e - A list of meaningful things that happen at a specific point in time. E.g. \u003ccode\u003ePage finished loading\u003c/code\u003e at \u003ccode\u003e2024-03-13T18:52:58.114561Z\u003c/code\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp\u003e\u003cstrong\u003eLinks\u003c/strong\u003e -A list of other span IDs (potentially in other traces) that are causally related to this span. This is useful for connecting operations that are asynchronous. For example, if someone buys an item from our online store, we might want to send them an email later. The email sending is triggered by the purchase request trace but isn't part of the same request, so we can connect them using a link.\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003eYou might be thinking, how do I know when to start a new span?\u003c/p\u003e\n\u003cp\u003eThere's no strict rule for the level of detail that would warrant starting a new span. You might have a single span for all the work done in one endpoint, or a span for each function call in your service. In general, the higher the level of detail, the more targeted and useful your traces will be, but a higher level of detail often comes with a larger upfront and ongoing maintenance cost. Different people handle this in various ways, and it often depends on how your service is instrumented. More on this topic later.\u003c/p\u003e\n\u003cp\u003eAnd that's pretty much it. A full trace looks something like this.\u003c/p\u003e\n\u003cimg alt=\"Trace X diagram with Span A as root, branching to Span B → Span D and Span C → Span E → Span G, plus Span F.\" src=\"/static/images/knowledge_base/traceX-with-spans.png\" style=\"background-color: #f8f9fa; padding: 10px; border-radius: 8px;\"\u003e\n\u003ch3 id=\"how-are-traces-generated\"\u003eHow are traces generated?\u003c/h3\u003e\n\u003cp\u003eAs we now know, traces are made up of a bunch of related spans. So how are these spans generated?\u003c/p\u003e\n\u003cp\u003eLet's begin with a simple example: a trace that consists of a single span without any children.\u003c/p\u003e\n\u003cp\u003eTo create a trace with a single span, we need to generate some identifiers, name the operation we're recording, track the duration, and then emit the span (we'll discuss where it goes later). In pseudo Go code, it might look like this.\u003c/p\u003e\n\u003cpre class=\"language-go\"\u003e\u003ccode class=\"language-go code-highlight\"\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Generate identifiers\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003etraceID \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e \u003cspan class=\"token function\"\u003egenerateTraceID\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003espanID \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e \u003cspan class=\"token function\"\u003egenerateSpanID\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003espanName \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e \u003cspan class=\"token string\"\u003e\"Important Operation To Record\"\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003estartTime \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e time\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eNow\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Perform the operation that we want to trace\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token function\"\u003edoOperation\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003eendTime \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e time\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eNow\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token function\"\u003eemitSpan\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003eSpan\u003cspan class=\"token punctuation\"\u003e{\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e TraceId\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e traceId\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e ParentSpanId\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e \u003cspan class=\"token boolean\"\u003enil\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e SpanName\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e spanName\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e \n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e SpanId\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e spanId\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e \n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e StartTime\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e startTime\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e \n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e EndTime\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e endTime\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token punctuation\"\u003e}\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eSimple enough right?\u003c/p\u003e\n\u003cp\u003eBut now we want to add child spans to give us a more granular view of what's happening in our service. To emit a child span, we need to know two things:\u003c/p\u003e\n\u003col\u003e\n\u003cli\u003e\n\u003cp\u003eThe traceId generated when we create the first span for the trace\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp\u003eThe spanId of the parent span\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ol\u003e\n\u003cp\u003eOkay, so now we need to pass around these two identifiers and we should be good to go. Let's pass around the traceId and current span id around in a \u003ca href=\"https://go.dev/blog/context\"\u003econtext\u003c/a\u003e. Now our pseudo span generation code looks something like this.\u003c/p\u003e\n\u003cpre class=\"language-go\"\u003e\u003ccode class=\"language-go code-highlight\"\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Get the traceId if it exists, otherwise generate one\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003etraceId \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e ctx\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eValue\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token string\"\u003e\"traceId\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token keyword\"\u003eif\u003c/span\u003e traceId \u003cspan class=\"token operator\"\u003e==\u003c/span\u003e \u003cspan class=\"token boolean\"\u003enil\u003c/span\u003e \u003cspan class=\"token punctuation\"\u003e{\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e traceId \u003cspan class=\"token operator\"\u003e=\u003c/span\u003e \u003cspan class=\"token function\"\u003egenerateTraceID\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token punctuation\"\u003e}\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Get the parent spanId if it exists\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003eparentSpanId \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e ctx\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eValue\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token string\"\u003e\"spanId\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Generate the new child spanId\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003espanId \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e \u003cspan class=\"token function\"\u003egenerateSpanID\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003espanName \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e \u003cspan class=\"token string\"\u003e\"Important Operation To Record\"\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003estartTime \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e time\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eNow\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Perform the operation that we want to trace\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Pass down the context with the new spanId incase we want to make a new child span where this span is the parent\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003ectxWithParentSpanId \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e context\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eWithValue\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003ectx\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e \u003cspan class=\"token string\"\u003e\"spanId\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e spanId\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e \n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token function\"\u003edoOperation\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003ectxWithParentSpanId\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003eendTime \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e time\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eNow\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token function\"\u003eemitSpan\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003eSpan\u003cspan class=\"token punctuation\"\u003e{\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e TraceId\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e traceId\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e ParentSpanId\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e parentSpanId\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e SpanName\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e spanName\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e \n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e SpanId\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e spanId\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e \n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e StartTime\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e startTime\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e \n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e EndTime\u003cspan class=\"token punctuation\"\u003e:\u003c/span\u003e endTime\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token punctuation\"\u003e}\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThis works well enough until we need to pass values across the network so that we can create child spans in different services.\u003c/p\u003e\n\u003cp\u003eDifferent protocols have different ways of doing this. For HTTP, \u003ca href=\"https://www.w3.org/TR/trace-context/#trace-context-http-headers-format\"\u003eW3C recommends\u003c/a\u003e that this information is passed around in the \u003ccode\u003etraceparent\u003c/code\u003e header. It's then up to the service on the other side of the request to pull this information out of http request and propagate it down through all the code handling the request. Giving us something like this for a system with 3 microservices that communicate via HTTP.\u003c/p\u003e\n\u003cimg alt=\"Sequence diagram of a client request traced through Service A, Service B, and Service C, with each generating child spans and returning the response to the client.\" src=\"/static/images/knowledge_base/detailed-flow-graph-distributed-tracing.png\" style=\"background-color: #f8f9fa; padding: 10px; border-radius: 8px;\"\u003e\n\u003cp\u003eSo now we need some code which adds the \u003ccode\u003etraceparent\u003c/code\u003e header to our http requests. It might look like something like this for a request made with the standard go http client\u003c/p\u003e\n\u003cpre class=\"language-go\"\u003e\u003ccode class=\"language-go code-highlight\"\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Get the traceId if it exists, otherwise generate one\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003etraceId \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e ctx\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eValue\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token string\"\u003e\"traceId\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token keyword\"\u003eif\u003c/span\u003e traceId \u003cspan class=\"token operator\"\u003e==\u003c/span\u003e \u003cspan class=\"token boolean\"\u003enil\u003c/span\u003e \u003cspan class=\"token punctuation\"\u003e{\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e traceId \u003cspan class=\"token operator\"\u003e=\u003c/span\u003e \u003cspan class=\"token function\"\u003egenerateTraceID\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token punctuation\"\u003e}\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Generate the new child spanId\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003espanId \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e \u003cspan class=\"token function\"\u003egenerateSpanID\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Get the parent spanId if it exists\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003eparentSpanId \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e ctx\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eValue\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token string\"\u003e\"spanId\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003espanName \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e \u003cspan class=\"token string\"\u003e\"HTTP Operation\"\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003estartTime \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e time\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eNow\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Create a new context with the spanId, indicating this span is now the parent for any child spans\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003ectxWithSpanId \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e context\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eWithValue\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003ectx\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e \u003cspan class=\"token string\"\u003e\"spanId\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e spanId\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e \n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Create an HTTP client and request\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003ehttpClient \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e \u003cspan class=\"token function\"\u003ecreateHttpClient\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003ehttpRequest \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e \u003cspan class=\"token function\"\u003ecreateHttpRequest\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token string\"\u003e\"http://my-upstream-microservice/api/endpoint\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Add the traceparent header to the HTTP request\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// The traceparent header format: traceId-parentSpanId (simplified)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003etraceparentValue \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e fmt\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eSprintf\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token string\"\u003e\"%s-%s\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e traceId\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e spanId\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003ehttpRequest\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003eHeader\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eAdd\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token string\"\u003e\"traceparent\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e traceparentValue\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token comment\"\u003e// Perform the HTTP call\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003eresponse \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e httpClient\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eDo\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003ehttpRequest\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e ctxWithSpanId\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eWe also need code to read the value of the HTTP header when we receive a request in our chosen web framework. This is how it might look using Go's standard library HTTP server.\u003c/p\u003e\n\u003cpre class=\"language-go\"\u003e\u003ccode class=\"language-go code-highlight\"\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token keyword\"\u003efunc\u003c/span\u003e \u003cspan class=\"token function\"\u003ehandler\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003ew http\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003eResponseWriter\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e r \u003cspan class=\"token operator\"\u003e*\u003c/span\u003ehttp\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003eRequest\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e \u003cspan class=\"token punctuation\"\u003e{\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e \u003cspan class=\"token comment\"\u003e// Extract the traceparent header\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e traceparentHeader \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e r\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003eHeader\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eGet\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token string\"\u003e\"traceparent\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e \n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e \u003cspan class=\"token comment\"\u003e// Parse the traceparent header\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e parts \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e strings\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eSplit\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003etraceparentHeader\u003cspan class=\"token punctuation\"\u003e,\u003c/span\u003e \u003cspan class=\"token string\"\u003e\"-\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e traceId \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e parts\u003cspan class=\"token punctuation\"\u003e[\u003c/span\u003e\u003cspan class=\"token number\"\u003e0\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e]\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e parentSpanId \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e parts\u003cspan class=\"token punctuation\"\u003e[\u003c/span\u003e\u003cspan class=\"token number\"\u003e1\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e]\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e \u003cspan class=\"token comment\"\u003e// Pass these around to the rest of our code\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token punctuation\"\u003e}\u003c/span\u003e\n\u003c/span\u003e\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eNow writing all of this code to instrument communication libraries and frameworks and emitting traces seems like a lot of work.\u003c/p\u003e\n\u003cp\u003eCan we automate this? Why yes, we can.\u003c/p\u003e\n\u003ch3 id=\"automated-distributed-tracing-instrumentation\"\u003eAutomated distributed tracing instrumentation\u003c/h3\u003e\n\u003cp\u003eThere are three broad buckets of automated distributed tracing instrumentation. We'll dive deeper into each of these in a subsequent post, but for now here's a quick overview of each of them.\u003c/p\u003e\n\u003ch4 id=\"compile-time-instrumentation\"\u003eCompile Time Instrumentation\u003c/h4\u003e\n\u003cp\u003eThis approach uses libraries or plugins added to existing libraries to incorporate instrumentation into the application. These plugins or libraries are included at compile time and require application level changes.\u003c/p\u003e\n\u003cp\u003eFor example, the \u003ca href=\"https://pkg.go.dev/go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin\"\u003eotelgin\u003c/a\u003e Go library offers middleware that automatically propagates trace metadata for all requests passing through the Gin HTTP server, similar to the process we coded by hand before.\u003c/p\u003e\n\u003cpre class=\"language-go\"\u003e\u003ccode class=\"language-go code-highlight\"\u003e\u003cspan class=\"code-line\"\u003er \u003cspan class=\"token operator\"\u003e:=\u003c/span\u003e gin\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eNew\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003er\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eUse\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003eotelgin\u003cspan class=\"token punctuation\"\u003e.\u003c/span\u003e\u003cspan class=\"token function\"\u003eMiddleware\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e(\u003c/span\u003e\u003cspan class=\"token string\"\u003e\"my_service_name\"\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\u003cspan class=\"token punctuation\"\u003e)\u003c/span\u003e\n\u003c/span\u003e\u003cspan class=\"code-line\"\u003e\u003cspan class=\"token operator\"\u003e...\u003c/span\u003e\n\u003c/span\u003e\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThis approach is effective across a wide variety of languages and libraries, offering the most flexibility but also being the most time-consuming.\u003c/p\u003e\n\u003ch4 id=\"automated-language-level-runtime-instrumentation\"\u003eAutomated Language Level Runtime Instrumentation\u003c/h4\u003e\n\u003cp\u003eThis approach injects instrumentation into the application at runtime. This is a great option for languages which have a dynamic execution environment like python, java or node.js. An example of tooling targeting this class of instrumentation is the \u003ca href=\"https://opentelemetry.io/docs/kubernetes/operator/automatic/\"\u003eOpenTelemetry kubernetes operator\u003c/a\u003e which injects instrumentation libraries into containers at startup depending on their language.\u003c/p\u003e\n\u003cp\u003eThe injected code automatically instruments a number of popular libraries in the specified language so as long as you're using one of the supported libraries then this is a great option which tends to work better than the the kernel level instrumentation and also doesn't require an application level change.\u003c/p\u003e\n\u003ch4 id=\"automated-kernel-runtime-instrumentation\"\u003eAutomated Kernel Runtime Instrumentation\u003c/h4\u003e\n\u003cp\u003eThis instrumentation approach adds instrumentation at the level of the Kernel. In linux these tools tend to be based on \u003ca href=\"https://ebpf.io/\"\u003eeBPF\u003c/a\u003e like \u003ca href=\"https://cilium.io/\"\u003eCilium\u003c/a\u003e or \u003ca href=\"https://github.com/grafana/beyla\"\u003eBeyla\u003c/a\u003e.\u003c/p\u003e\n\u003cp\u003eThey intercept \u003ca href=\"https://man7.org/linux/man-pages/man2/syscalls.2.html\"\u003esyscalls\u003c/a\u003e and automatically emit spans. These tools don't require services (or their runtime) to be modified in any way and work with all languages so they're a great choice to get started with. However, the instrumentation is often coarse as it only has access to a small amount of information about the service.\u003c/p\u003e\n\u003cp\u003eOkay, now we know what a trace is, how spans link together and how we can get our applications instrumented and producing telemetry without too much manual work.\u003c/p\u003e\n\u003cp\u003eThe next step is to get those spans exported somewhere they can be useful.\u003c/p\u003e\n\u003ch3 id=\"how-are-traces-actually-emitted-and-collected\"\u003eHow are traces actually emitted and collected?\u003c/h3\u003e\n\u003cp\u003eIn order for trace information to be useful, we need to have a centralised location where we can query them. This location could be a commercial cloud observability provider like \u003ca href=\"https://www.datadoghq.com/\"\u003eDataDog\u003c/a\u003e or an internally hosted trace store like \u003ca href=\"https://www.jaegertracing.io/\"\u003eJaeger\u003c/a\u003e or \u003ca href=\"https://github.com/grafana/tempo\"\u003eTempo\u003c/a\u003e.\u003c/p\u003e\n\u003cp\u003eWhen emitting your trace data to a trace store you have a couple of choices.\u003c/p\u003e\n\u003col\u003e\n\u003cli\u003e\n\u003cp\u003eShip the data to the store directly from the span creating service\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp\u003eShip the data to a collector which then ships it to the store\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ol\u003e\n\u003cp\u003eGenerally, using an intermediate collector like the \u003ca href=\"https://opentelemetry.io/docs/collector/\"\u003eOpenTelemetry Collector\u003c/a\u003e is a better choice as it gives you access to more advanced techniques like \u003ca href=\"https://opentelemetry.io/blog/2022/tail-sampling/\"\u003etail sampling\u003c/a\u003e and is more robust as your production services can offload the data quickly without worrying about retries, batching and a bunch of other concerns.\u003c/p\u003e\n\u003cp\u003eEach service establishes a connection with a collector and sends spans over the network as they're generated, open telemetry defines the \u003ca href=\"https://opentelemetry.io/docs/specs/otlp/\"\u003eOTLP\u003c/a\u003e protocol for sending telemetry which sits on top of gRPC or HTTP.\u003c/p\u003e\n\u003cimg alt=\"Diagram showing Service A, B, and C sending traces via OTLP to an OTel Collector, which forwards them with TraceStoreProtocol to a Trace Store.\" src=\"/static/images/knowledge_base/otlp-collector-trace-store.png\" style=\"background-color: #f8f9fa; padding: 10px; border-radius: 8px;\"\u003e\n\u003cp\u003eNow we have our services producing spans, they're sent to the collector which forwards them to the trace store. The trace store indexes these spans by the traceId and a bunch of other characteristics so you can run queries to find interesting behaviour in your systems.\u003c/p\u003e\n\u003cp\u003eNow you have a solid idea of what traces are, how they're generated and an idea of what they can be used for.\u003c/p\u003e\n\u003ch3 id=\"future-posts\"\u003eFuture posts\u003c/h3\u003e\n\u003cp\u003eI'll be writing a new blog post each week on this space so subscribe to the newsletter if you found this interesting and let me know what topics you'd like to see!\u003c/p\u003e"])</script><script>self.__next_f.push([1,"12:[\"$\",\"$L27\",null,{\"children\":[[\"$\",\"div\",null,{\"className\":\"min-h-screen bg-[var(--article-bg-secondary)] relative transition-colors duration-300\",\"children\":[[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"{\\\"@context\\\":\\\"https://schema.org\\\",\\\"@type\\\":\\\"BlogPosting\\\",\\\"headline\\\":\\\"Distributed Tracing: A Whistle Stop Tour\\\",\\\"description\\\":\\\"Know enough to be dangerous in 10 minutes\\\",\\\"keywords\\\":\\\"distributed tracing, tracing, observability, microservices, OpenTelemetry, Jaeger, Zipkin\\\",\\\"mainEntityOfPage\\\":{\\\"@type\\\":\\\"WebPage\\\",\\\"@id\\\":\\\"https://metoro.io/blog/distributed-tracing-whistle-stop-tour\\\"},\\\"publisher\\\":{\\\"@type\\\":\\\"Organization\\\",\\\"name\\\":\\\"Metoro\\\",\\\"logo\\\":{\\\"@type\\\":\\\"ImageObject\\\",\\\"url\\\":\\\"https://metoro.io/static/images/Metoro_16_9.svg\\\"}},\\\"datePublished\\\":\\\"2024-03-13\\\",\\\"dateModified\\\":\\\"2024-03-13\\\",\\\"author\\\":{\\\"@type\\\":\\\"Person\\\",\\\"name\\\":\\\"Chris Battarbee\\\",\\\"url\\\":\\\"https://metoro.io/authors/chris-battarbee\\\",\\\"description\\\":\\\"CEO, ex-Palantir Senior Software Engineer on Compute and Kubernetes\\\"},\\\"timeRequired\\\":\\\"PT11M\\\"}\"}}],[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"{\\\"@context\\\":\\\"https://schema.org\\\",\\\"@type\\\":\\\"BreadcrumbList\\\",\\\"itemListElement\\\":[{\\\"@type\\\":\\\"ListItem\\\",\\\"position\\\":1,\\\"name\\\":\\\"Home\\\",\\\"item\\\":\\\"https://metoro.io/\\\"},{\\\"@type\\\":\\\"ListItem\\\",\\\"position\\\":2,\\\"name\\\":\\\"Blog\\\",\\\"item\\\":\\\"https://metoro.io/blog\\\"},{\\\"@type\\\":\\\"ListItem\\\",\\\"position\\\":3,\\\"name\\\":\\\"Distributed Tracing: A Whistle Stop Tour\\\",\\\"item\\\":\\\"https://metoro.io/blog/distributed-tracing-whistle-stop-tour\\\"}]}\"}}],[\"$\",\"aside\",null,{\"className\":\"hidden lg:block fixed left-4 top-1/2 -translate-y-1/2 z-20\",\"children\":[\"$\",\"$L28\",null,{\"toc\":[{\"id\":\"what-problem-do-distributed-traces-solve\",\"title\":\"What problem do distributed traces solve?\",\"level\":2},{\"id\":\"how-do-distributed-traces-work\",\"title\":\"How do distributed traces work?\",\"level\":2},{\"id\":\"what-is-a-trace\",\"title\":\"What is a trace?\",\"level\":3},{\"id\":\"how-are-traces-generated\",\"title\":\"How are traces generated?\",\"level\":3},{\"id\":\"automated-distributed-tracing-instrumentation\",\"title\":\"Automated distributed tracing instrumentation\",\"level\":3},{\"id\":\"compile-time-instrumentation\",\"title\":\"Compile Time Instrumentation\",\"level\":4},{\"id\":\"automated-language-level-runtime-instrumentation\",\"title\":\"Automated Language Level Runtime Instrumentation\",\"level\":4},{\"id\":\"automated-kernel-runtime-instrumentation\",\"title\":\"Automated Kernel Runtime Instrumentation\",\"level\":4},{\"id\":\"how-are-traces-actually-emitted-and-collected\",\"title\":\"How are traces actually emitted and collected?\",\"level\":3},{\"id\":\"future-posts\",\"title\":\"Future posts\",\"level\":3}]}]}],[\"$\",\"div\",null,{\"className\":\"relative z-10 mx-auto max-w-4xl px-6 lg:px-8 pb-24 pt-8\",\"children\":[[\"$\",\"nav\",null,{\"className\":\"flex items-center text-sm text-[var(--article-text-secondary)] mb-8 transition-colors\",\"aria-label\":\"Breadcrumb\",\"children\":[[\"$\",\"$L29\",null,{\"href\":\"/\",\"className\":\"hover:text-[var(--article-heading)] transition-colors\",\"children\":\"Home\"}],[\"$\",\"span\",null,{\"className\":\"mx-2 text-[var(--article-text-secondary)]\",\"children\":\"/\"}],[\"$\",\"$L29\",null,{\"href\":\"/blog\",\"className\":\"hover:text-[var(--article-heading)] transition-colors\",\"children\":\"Blog\"}],[[\"$\",\"span\",\"distributed-tracing-whistle-stop-tour\",{\"children\":[[\"$\",\"span\",null,{\"className\":\"mx-2 text-[var(--article-text-secondary)]\",\"children\":\"/\"}],[\"$\",\"span\",null,{\"className\":\"text-[var(--article-heading)] capitalize transition-colors\",\"children\":\"distributed tracing whistle stop tour\"}]]}]]]}],[\"$\",\"article\",null,{\"children\":[[\"$\",\"header\",null,{\"className\":\"mb-10\",\"children\":[[\"$\",\"$L2a\",null,{\"children\":[\"$\",\"h1\",null,{\"className\":\"font-sora font-semibold text-[28px] sm:text-[32px] lg:text-[36px] tracking-[-0.5px] leading-[1.2] text-[var(--article-heading)] mb-4 transition-colors\",\"children\":\"Distributed Tracing: A Whistle Stop Tour\"}]}],[\"$\",\"p\",null,{\"className\":\"font-sans font-normal text-[var(--article-text)] text-[16px] sm:text-[18px] leading-[1.6] mb-5 max-w-[800px] transition-colors\",\"children\":\"Know enough to be dangerous in 10 minutes\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center text-sm text-[var(--article-text-secondary)] space-x-4 flex-wrap font-sora transition-colors\",\"children\":[[\"$\",\"a\",null,{\"href\":\"/authors/chris-battarbee\",\"children\":[\"By \",[\"$\",\"span\",null,{\"className\":\"text-[var(--article-link)]\",\"children\":\"Chris Battarbee\"}]]}],[\"$\",\"div\",null,{\"className\":\"flex items-center space-x-1\",\"children\":[[\"$\",\"span\",null,{\"children\":\"Published:\"}],[\"$\",\"time\",null,{\"dateTime\":\"2024-03-13\",\"children\":\"March 13, 2024\"}]]}],false,[\"$\",\"div\",null,{\"className\":\"flex items-center space-x-1\",\"children\":[\"$\",\"span\",null,{\"children\":[11,\" min read\"]}]}]]}]]}],[\"$\",\"$L2b\",null,{\"htmlContent\":\"$2c\",\"className\":\"knowledge-base-content\"}],[\"$\",\"section\",null,{\"className\":\"mt-16 flex flex-col gap-5 rounded-[16px] border border-[var(--article-border)] bg-[var(--article-bg)] p-6 transition-colors sm:flex-row sm:items-center\",\"children\":[[\"$\",\"$L2d\",null,{\"src\":\"/static/images/avatars/chris-battarbee.jpg\",\"alt\":\"Chris Battarbee avatar\",\"width\":64,\"height\":64,\"className\":\"h-16 w-16 flex-shrink-0 rounded-full border border-[var(--article-border)] object-cover\"}],[\"$\",\"div\",null,{\"children\":[[\"$\",\"div\",null,{\"className\":\"font-sora text-[12px] uppercase tracking-[0.14em] text-[var(--article-text-secondary)] transition-colors\",\"children\":\"Written by\"}],[\"$\",\"a\",null,{\"href\":\"/authors/chris-battarbee\",\"className\":\"mt-1 inline-block font-sora text-[18px] font-semibold text-[var(--article-link)] transition-colors hover:text-[var(--article-link-hover)]\",\"children\":\"Chris Battarbee\"}],[\"$\",\"p\",null,{\"className\":\"mt-2 max-w-[620px] font-sans text-[15px] leading-[1.6] text-[var(--article-text)] transition-colors\",\"children\":\"CEO, ex-Palantir Senior Software Engineer on Compute and Kubernetes\"}]]}]]}],[\"$\",\"section\",null,{\"className\":\"mt-16 border-t border-[var(--article-border)] pt-10 transition-colors\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mb-6 max-w-2xl\",\"children\":[[\"$\",\"h2\",null,{\"className\":\"font-sora font-semibold text-[24px] sm:text-[28px] tracking-[-0.5px] text-[var(--article-heading)] transition-colors\",\"children\":\"Related reading\"}],[\"$\",\"p\",null,{\"className\":\"mt-2 font-sans text-[15px] leading-[1.6] text-[var(--article-text-secondary)] transition-colors\",\"children\":\"More Metoro articles that deepen the same topic from another angle.\"}]]}],[\"$\",\"div\",null,{\"className\":\"grid grid-cols-1 gap-4 md:grid-cols-3\",\"children\":[[\"$\",\"$L29\",\"/blog/microservice-interview-questions\",{\"href\":\"/blog/microservice-interview-questions\",\"className\":\"group flex h-full flex-col rounded-[16px] border border-[var(--article-border)] bg-[var(--article-bg)] p-5 transition-colors hover:border-[var(--article-link)]\",\"children\":[[\"$\",\"time\",null,{\"dateTime\":\"2024-03-18\",\"className\":\"font-sora text-[11px] uppercase tracking-[0.18em] text-[var(--article-text-secondary)] transition-colors\",\"children\":\"Mar 18, 2024\"}],[\"$\",\"h3\",null,{\"className\":\"mt-3 font-sora text-[19px] font-semibold leading-[1.35] text-[var(--article-heading)] transition-colors group-hover:text-[var(--article-link)]\",\"children\":\"Microservice Interview Questions: The basics\"}],[\"$\",\"p\",null,{\"className\":\"mt-3 flex-1 font-sans text-[15px] leading-[1.6] text-[var(--article-text)] transition-colors\",\"children\":\"Detailed answers to common questions about microservices, starting from the basics and getting into the details\"}],[\"$\",\"span\",null,{\"className\":\"mt-5 font-sora text-[14px] text-[var(--article-link)] transition-colors group-hover:text-[var(--article-link-hover)]\",\"children\":\"Read article →\"}]]}],[\"$\",\"$L29\",\"/blog/how-metoro-uses-ebpf\",{\"href\":\"/blog/how-metoro-uses-ebpf\",\"className\":\"group flex h-full flex-col rounded-[16px] border border-[var(--article-border)] bg-[var(--article-bg)] p-5 transition-colors hover:border-[var(--article-link)]\",\"children\":[[\"$\",\"time\",null,{\"dateTime\":\"2026-01-27\",\"className\":\"font-sora text-[11px] uppercase tracking-[0.18em] text-[var(--article-text-secondary)] transition-colors\",\"children\":\"Jan 27, 2026\"}],[\"$\",\"h3\",null,{\"className\":\"mt-3 font-sora text-[19px] font-semibold leading-[1.35] text-[var(--article-heading)] transition-colors group-hover:text-[var(--article-link)]\",\"children\":\"How Metoro Uses eBPF for Zero-Instrumentation Observability\"}],[\"$\",\"p\",null,{\"className\":\"mt-3 flex-1 font-sans text-[15px] leading-[1.6] text-[var(--article-text)] transition-colors\",\"children\":\"A technical deep-dive into how Metoro captures L7 protocol traffic and intercepts TLS-encrypted data using eBPF, enabling automatic observability without code changes\"}],[\"$\",\"span\",null,{\"className\":\"mt-5 font-sora text-[14px] text-[var(--article-link)] transition-colors group-hover:text-[var(--article-link-hover)]\",\"children\":\"Read article →\"}]]}],[\"$\",\"$L29\",\"/blog/kubernetes-observability\",{\"href\":\"/blog/kubernetes-observability\",\"className\":\"group flex h-full flex-col rounded-[16px] border border-[var(--article-border)] bg-[var(--article-bg)] p-5 transition-colors hover:border-[var(--article-link)]\",\"children\":[[\"$\",\"time\",null,{\"dateTime\":\"2026-02-03\",\"className\":\"font-sora text-[11px] uppercase tracking-[0.18em] text-[var(--article-text-secondary)] transition-colors\",\"children\":\"Feb 3, 2026\"}],[\"$\",\"h3\",null,{\"className\":\"mt-3 font-sora text-[19px] font-semibold leading-[1.35] text-[var(--article-heading)] transition-colors group-hover:text-[var(--article-link)]\",\"children\":\"Kubernetes Observability: The Complete Guide\"}],[\"$\",\"p\",null,{\"className\":\"mt-3 flex-1 font-sans text-[15px] leading-[1.6] text-[var(--article-text)] transition-colors\",\"children\":\"Learn what Kubernetes observability is and how to implement effective observability for your k8s clusters.\"}],[\"$\",\"span\",null,{\"className\":\"mt-5 font-sora text-[14px] text-[var(--article-link)] transition-colors group-hover:text-[var(--article-link-hover)]\",\"children\":\"Read article →\"}]]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"mt-16 pt-8 border-t border-[var(--article-border)] transition-colors\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex justify-between items-center\",\"children\":[[\"$\",\"$L29\",null,{\"href\":\"/blog\",\"className\":\"font-sora font-normal text-[var(--article-link)] text-[16px] hover:text-[var(--article-link-hover)] transition-colors flex items-center gap-2\",\"children\":[\"← Back to \",\"Blog\"]}],[[\"$\",\"$L29\",\"0\",{\"href\":\"/docs\",\"className\":\"font-sora font-normal text-[16px] transition-colors flex items-center gap-2 text-[var(--article-link)] hover:text-[var(--article-link-hover)]\",\"children\":[\"Read the Technical docs\",\" →\"]}]]]}]}]]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"bg-[#05080d]\",\"children\":[\"$\",\"$L2e\",null,{}]}]]}]\n"])</script><script>self.__next_f.push([1,"24:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"Distributed Tracing: A Whistle Stop Tour\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"Know enough to be dangerous in 10 minutes\"}],[\"$\",\"meta\",\"4\",{\"name\":\"author\",\"content\":\"Chris Battarbee\"}],[\"$\",\"meta\",\"5\",{\"name\":\"keywords\",\"content\":\"observability,kubernetes,ebpf,monitoring,platform\"}],[\"$\",\"meta\",\"6\",{\"name\":\"robots\",\"content\":\"index, follow\"}],[\"$\",\"link\",\"7\",{\"rel\":\"canonical\",\"href\":\"https://metoro.io/blog/distributed-tracing-whistle-stop-tour\"}],[\"$\",\"link\",\"8\",{\"rel\":\"alternate\",\"hrefLang\":\"en\",\"href\":\"https://metoro.io/blog/distributed-tracing-whistle-stop-tour\"}],[\"$\",\"meta\",\"9\",{\"property\":\"og:title\",\"content\":\"Distributed Tracing: A Whistle Stop Tour\"}],[\"$\",\"meta\",\"10\",{\"property\":\"og:description\",\"content\":\"Know enough to be dangerous in 10 minutes\"}],[\"$\",\"meta\",\"11\",{\"property\":\"og:url\",\"content\":\"https://metoro.io/blog/distributed-tracing-whistle-stop-tour\"}],[\"$\",\"meta\",\"12\",{\"property\":\"og:image\",\"content\":\"https://metoro.io/static/images/logos/MetoroLogo.png\"}],[\"$\",\"meta\",\"13\",{\"property\":\"og:image:width\",\"content\":\"1250\"}],[\"$\",\"meta\",\"14\",{\"property\":\"og:image:height\",\"content\":\"1250\"}],[\"$\",\"meta\",\"15\",{\"property\":\"og:image:alt\",\"content\":\"Distributed Tracing: A Whistle Stop Tour\"}],[\"$\",\"meta\",\"16\",{\"property\":\"og:type\",\"content\":\"article\"}],[\"$\",\"meta\",\"17\",{\"property\":\"article:published_time\",\"content\":\"2024-03-13\"}],[\"$\",\"meta\",\"18\",{\"property\":\"article:modified_time\",\"content\":\"2024-03-13\"}],[\"$\",\"meta\",\"19\",{\"property\":\"article:author\",\"content\":\"Chris Battarbee\"}],[\"$\",\"meta\",\"20\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"21\",{\"name\":\"twitter:site\",\"content\":\"@metoroio\"}],[\"$\",\"meta\",\"22\",{\"name\":\"twitter:title\",\"content\":\"Distributed Tracing: A Whistle Stop Tour\"}],[\"$\",\"meta\",\"23\",{\"name\":\"twitter:description\",\"content\":\"Know enough to be dangerous in 10 minutes\"}],[\"$\",\"meta\",\"24\",{\"name\":\"twitter:image\",\"content\":\"https://metoro.io/static/images/logos/MetoroLogo.png\"}],[\"$\",\"link\",\"25\",{\"rel\":\"icon\",\"href\":\"/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"26\",{\"name\":\"next-size-adjust\"}]]\n"])</script><script>self.__next_f.push([1,"11:null\n"])</script></body></html>