1459 lines
120 KiB
HTML
1459 lines
120 KiB
HTML
<!DOCTYPE html><html lang="en"><head><!-- Global Metadata --><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" type="image/svg+xml" href="/favicon-dark.svg" media="(prefers-color-scheme: dark)"><link rel="icon" type="image/svg+xml" href="/favicon-light.svg" media="(prefers-color-scheme: light)"><link rel="alternate" type="application/rss+xml" title="A Practitioner's Guide to Wide Events | Jeremy Morrell" href="https://jeremymorrell.dev/rss.xml"><link rel="icon" type="image/x-icon" href="/favicon-light.svg"><meta name="generator" content="Astro v7.0.7"><!-- Font preloads --><!-- <link rel="preload" href={inter400} as="font" type="font/woff2" crossorigin />
|
||
<link rel="preload" href={inter600} as="font" type="font/woff2" crossorigin />
|
||
<link rel="preload" href={lora400} as="font" type="font/woff2" crossorigin />
|
||
<link rel="preload" href={lora600} as="font" type="font/woff2" crossorigin /> --><!-- Canonical URL --><link rel="canonical" href="https://jeremymorrell.dev/blog/a-practitioners-guide-to-wide-events/"><!-- Primary Meta Tags --><title>A Practitioner's Guide to Wide Events | Jeremy Morrell</title><meta name="title" content="A Practitioner's Guide to Wide Events | Jeremy Morrell"><meta name="description" content="The existing articles on Wide Events define the concept well but leave the implementation details to the reader."><!-- Open Graph --><meta property="og:type" content="article"><meta property="og:site_name" content="Jeremy Morrell"><meta property="og:locale" content="en_US"><meta property="og:url" content="https://jeremymorrell.dev/blog/a-practitioners-guide-to-wide-events/"><meta property="og:title" content="A Practitioner's Guide to Wide Events | Jeremy Morrell"><meta property="og:description" content="The existing articles on Wide Events define the concept well but leave the implementation details to the reader."><meta property="og:image" content="https://jeremymorrell.dev/og/blog/a-practitioners-guide-to-wide-events.png"><meta property="og:image:alt" content="A Practitioner's Guide to Wide Events | Jeremy Morrell"><meta property="article:published_time" content="2024-10-22T00:00:00.000Z"><meta property="article:author" content="Jeremy Morrell"><!-- Twitter --><meta name="twitter:card" content="summary_large_image"><meta name="twitter:url" content="https://jeremymorrell.dev/blog/a-practitioners-guide-to-wide-events/"><meta name="twitter:title" content="A Practitioner's Guide to Wide Events | Jeremy Morrell"><meta name="twitter:description" content="The existing articles on Wide Events define the concept well but leave the implementation details to the reader."><meta name="twitter:image" content="https://jeremymorrell.dev/og/blog/a-practitioners-guide-to-wide-events.png"><script>
|
||
(function () {
|
||
// @ts-expect-error - counterscale is not typed
|
||
window.counterscale = {
|
||
q: [["set", "siteId", window.location.hostname], ["trackPageview"]],
|
||
};
|
||
})();
|
||
</script><script>
|
||
function init() {
|
||
applyTheme();
|
||
onScroll();
|
||
|
||
const toggleThemeButton = document.getElementById("themeToggle");
|
||
toggleThemeButton?.addEventListener("click", () => {
|
||
const dark = !document.documentElement.classList.contains("dark");
|
||
localStorage.setItem("theme", dark ? "dark" : "light");
|
||
toggleTheme(dark);
|
||
});
|
||
|
||
document.addEventListener("scroll", onScroll);
|
||
}
|
||
|
||
function onScroll() {
|
||
if (window.scrollY > 0) {
|
||
document.documentElement.classList.add("scrolled");
|
||
} else {
|
||
document.documentElement.classList.remove("scrolled");
|
||
}
|
||
}
|
||
|
||
// @ts-expect-error - this code is not typed
|
||
function toggleTheme(dark) {
|
||
const css = document.createElement("style");
|
||
|
||
css.appendChild(
|
||
document.createTextNode(
|
||
`* {
|
||
-webkit-transition: none !important;
|
||
-moz-transition: none !important;
|
||
-o-transition: none !important;
|
||
-ms-transition: none !important;
|
||
transition: none !important;
|
||
}
|
||
`
|
||
)
|
||
);
|
||
|
||
document.head.appendChild(css);
|
||
|
||
if (dark) {
|
||
document.documentElement.classList.add("dark");
|
||
document.documentElement.classList.remove("light");
|
||
} else {
|
||
document.documentElement.classList.remove("dark");
|
||
document.documentElement.classList.add("light");
|
||
}
|
||
|
||
window.getComputedStyle(css).opacity;
|
||
document.head.removeChild(css);
|
||
}
|
||
|
||
function applyTheme() {
|
||
const userTheme = localStorage.theme;
|
||
|
||
if (userTheme === "light" || userTheme === "dark") {
|
||
toggleTheme(userTheme === "dark");
|
||
} else {
|
||
toggleTheme(window.matchMedia("(prefers-color-scheme: dark)").matches);
|
||
}
|
||
}
|
||
|
||
// Keep the page in sync when the OS theme changes mid-session (e.g. macOS
|
||
// auto dark mode at sunset). applyTheme still lets an explicit toggle
|
||
// choice in localStorage win.
|
||
window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change", () => applyTheme());
|
||
|
||
// Pages restored from the back/forward cache don't re-run scripts, so
|
||
// re-check the theme when that happens.
|
||
window.addEventListener("pageshow", (event) => {
|
||
if (event.persisted) applyTheme();
|
||
});
|
||
|
||
// Theme toggled in another tab.
|
||
window.addEventListener("storage", (event) => {
|
||
if (event.key === "theme") applyTheme();
|
||
});
|
||
|
||
document.addEventListener("DOMContentLoaded", () => init());
|
||
document.addEventListener("astro:after-swap", () => init());
|
||
applyTheme();
|
||
</script><script id="counterscale-script" src="https://counterscale.jeremymorrell.workers.dev/tracker.js" defer></script><link rel="stylesheet" href="/_astro/PageLayout.BBDlKKm3.css"></head><body><header class="py-5"><div class="mx-auto max-w-screen-md px-5"><div class="flex flex-wrap items-center justify-between gap-y-1 text-sm sm:text-base"><a href="/" target="_self" class="pr-2 py-0.5 rounded-md flex items-center"><div class="font-semibold whitespace-nowrap">Jeremy Morrell</div></a><nav class="flex flex-nowrap"><a href="/blog" target="_self" class="px-1 sm:px-2 py-0.5 rounded-md flex items-center transition-colors hover:text-black hover:dark:text-white hover:bg-black/5 hover:dark:bg-white/15"> blog </a><a href="/notes" target="_self" class="px-1 sm:px-2 py-0.5 rounded-md flex items-center transition-colors hover:text-black hover:dark:text-white hover:bg-black/5 hover:dark:bg-white/15"> notes </a><a href="/about" target="_self" class="px-1 sm:px-2 py-0.5 rounded-md flex items-center transition-colors hover:text-black hover:dark:text-white hover:bg-black/5 hover:dark:bg-white/15"> about </a><details class="relative flex items-stretch" data-rss-menu><summary class="flex cursor-pointer list-none items-center rounded-md px-1 py-0.5 transition-colors hover:bg-black/5 hover:text-black sm:px-2 dark:hover:bg-white/15 dark:hover:text-white [&::-webkit-details-marker]:hidden">rss</summary><ul class="absolute top-full right-0 z-20 mt-2 min-w-32 space-y-0.5 rounded-lg border border-black/10 bg-stone-100 p-1.5 text-sm shadow-lg dark:border-white/15 dark:bg-stone-900 sm:text-base"><li><a href="/blog/rss.xml" class="block rounded-md px-3 py-1.5 transition-colors hover:bg-black/5 hover:text-black dark:hover:bg-white/15 dark:hover:text-white">Blog</a></li><li><a href="/rss.xml" class="block rounded-md px-3 py-1.5 transition-colors hover:bg-black/5 hover:text-black dark:hover:bg-white/15 dark:hover:text-white">Everything</a></li></ul></details><a target="_self" class="px-1 sm:px-2 py-0.5 rounded-md flex items-center transition-colors hover:text-black hover:dark:text-white hover:bg-black/5 hover:dark:bg-white/15"><button id="themeToggle" class="group size-8 flex items-center justify-center rounded-full" data-astro-cid-rknshgfe><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="sun group-hover:stroke-black group-hover:dark:stroke-white" data-astro-cid-rknshgfe><circle cx="12" cy="12" r="5" data-astro-cid-rknshgfe></circle><line x1="12" y1="1" x2="12" y2="3" data-astro-cid-rknshgfe></line><line x1="12" y1="21" x2="12" y2="23" data-astro-cid-rknshgfe></line><line x1="4.22" y1="4.22" x2="5.64" y2="5.64" data-astro-cid-rknshgfe></line><line x1="18.36" y1="18.36" x2="19.78" y2="19.78" data-astro-cid-rknshgfe></line><line x1="1" y1="12" x2="3" y2="12" data-astro-cid-rknshgfe></line><line x1="21" y1="12" x2="23" y2="12" data-astro-cid-rknshgfe></line><line x1="4.22" y1="19.78" x2="5.64" y2="18.36" data-astro-cid-rknshgfe></line><line x1="18.36" y1="5.64" x2="19.78" y2="4.22" data-astro-cid-rknshgfe></line></svg><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="moon group-hover:stroke-black group-hover:dark:stroke-white" data-astro-cid-rknshgfe><path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z" data-astro-cid-rknshgfe></path></svg></button></a></nav></div></div></header><script type="module">document.documentElement.dataset.rssMenuListeners||(document.documentElement.dataset.rssMenuListeners=`true`,document.addEventListener(`click`,e=>{document.querySelectorAll(`details[data-rss-menu][open]`).forEach(t=>{e.target instanceof Node&&!t.contains(e.target)&&(t.open=!1)})}),document.addEventListener(`keydown`,e=>{e.key===`Escape`&&document.querySelectorAll(`details[data-rss-menu][open]`).forEach(e=>{e.open=!1,e.querySelector(`summary`)?.focus()})}));</script><main><div class="mx-auto max-w-screen-md px-5"><header class="measure mt-12 mb-10"><div class="text-sm tracking-wide text-black/50 dark:text-white/50"><time datetime="2024-10-22T00:00:00.000Z">Oct 22, 2024</time></div><h1 class="mt-2 text-3xl md:text-4xl font-semibold tracking-tight text-balance text-black dark:text-white">A Practitioner's Guide to Wide Events</h1></header><article class="measure"><p>Adopting Wide Event-style instrumentation has been one of the highest-leverage changes I’ve made
|
||
in my engineering career. The feedback loop on all my changes tightened and debugging systems
|
||
became so much easier. Systems that were scary to work on suddenly seemed a lot more manageable.</p>
|
||
<p>Lately there have been a lot of good blog posts on what “Wide Events” mean and why they are
|
||
important. Here are some of my recent favorites:</p>
|
||
<ul>
|
||
<li><a href="https://isburmistrov.substack.com/p/all-you-need-is-wide-events-not-metrics">All you need is Wide Events, not “Metrics, Logs and Traces”</a> by <a href="https://bsky.app/profile/isburmistrov.bsky.social">Ivan Burmistrov</a></li>
|
||
<li><a href="https://boristane.com/blog/observability-wide-events-101/">Observability wide events 101</a> by
|
||
<a href="https://twitter.com/boristane">Boris Tane</a></li>
|
||
<li><a href="https://charity.wtf/2024/08/07/is-it-time-to-version-observability-signs-point-to-yes/">Is it time to version Observability? (Signs point to yes)</a> by <a href="https://bsky.app/profile/mipsytipsy.bsky.social">Charity Majors</a></li>
|
||
</ul>
|
||
<p>The tl;dr is that for each unit-of-work in your system (usually, but not always an HTTP request / response)
|
||
you emit one “event” with all of the information you can collect about that work. “Event” is an over-loaded
|
||
term in telemetry so replace that with “log line” or “span” if you like. <a href="https://jeremymorrell.dev/blog/minimal-js-tracing/">They are all effectively the same
|
||
thing</a>.</p>
|
||
<p><a href="https://bsky.app/profile/mipsytipsy.bsky.social">Charity Majors</a> has been promoting this approach lately under the
|
||
name <a href="https://www.honeycomb.io/blog/one-key-difference-observability1dot0-2dot0">“Observability 2.0”</a>, creating some
|
||
new momentum around the concept, however, it is <em>not</em> a new idea. <a href="https://twitter.com/brandur">Brandur Leach</a> wrote
|
||
about “Canonical Log Lines” both on <a href="https://brandur.org/canonical-log-lines">his own blog in 2016</a> and
|
||
<a href="https://stripe.com/blog/canonical-log-lines">as used by Stripe in 2019</a>. And <a href="https://aws.amazon.com/builders-library/instrumenting-distributed-systems-for-operational-visibility/#Request_log_best_practices">AWS has recommended it as a best-practice for ages</a>.</p>
|
||
<h2 id="okay-i-think-i-get-the-idea-but-how-do-i-do-wide-events">Okay… I think I get the idea… but how do I do “wide events”?<a class="heading-anchor" aria-label="Link to this section" href="#okay-i-think-i-get-the-idea-but-how-do-i-do-wide-events">#</a></h2>
|
||
<p>This is where I find a lot of developers get tripped up. The idea sounds good in theory,
|
||
and we should totally try that one day! But I have this stack of features to ship, that
|
||
bug that’s been keeping me up at night, and 30 new AI tools that came out
|
||
yesterday to learn about. And like… where do you even start? What data should I add?</p>
|
||
<p>Like anything in software, there are a lot of options for how to approach this, but I’ll talk
|
||
through one approach that has worked for me.</p>
|
||
<p>We’ll cover how to approach this in tooling and code, an <strong>extensive</strong> list of attributes to add,
|
||
and I’ll respond to some frequent objections that come up when discussing this approach.</p>
|
||
<p>For this post we’ll focus on web services, but you would apply a similar approach to any workload.</p>
|
||
<h2 id="choose-your-tools">Choose your tools<a class="heading-anchor" aria-label="Link to this section" href="#choose-your-tools">#</a></h2>
|
||
<p>We will need some way to instrument your code (traces or structured log lines) and somewhere to
|
||
send the instrumentation to in order to query and visualize it.</p>
|
||
<p>This approach is best paired with a tool that lets you query your data in quick iterations.
|
||
I like <a href="https://www.honeycomb.io/">Honeycomb</a> for this, but any Observability tool backed by
|
||
a modern OLAP database is likely going to work in a pinch.</p>
|
||
<ul>
|
||
<li><a href="https://www.honeycomb.io/">Honeycomb</a> has <a href="https://www.honeycomb.io/resources/why-we-built-our-own-distributed-column-store">Retriever</a></li>
|
||
<li><a href="https://www.datadoghq.com/">DataDog</a> has <a href="https://www.datadoghq.com/blog/engineering/introducing-husky/">Husky</a></li>
|
||
<li><a href="https://newrelic.com/">New Relic</a> has <a href="https://docs.newrelic.com/docs/data-apis/get-started/nrdb-horsepower-under-hood/">NRDB</a></li>
|
||
<li><a href="https://baselime.io/">Baselime</a> uses <a href="https://boristane.com/talks/observability-with-clickhouse/">ClickHouse</a></li>
|
||
<li><a href="https://signoz.io/">SigNoz</a> uses <a href="https://clickhouse.com/blog/signoz-observability-solution-with-clickhouse-and-open-telemetry">ClickHouse</a></li>
|
||
</ul>
|
||
<p>Honeycomb, New Relic, and DataDog built their own columnar <a href="https://aws.amazon.com/compare/the-difference-between-olap-and-oltp/">OLAP</a> data stores,
|
||
though now with the availability of <a href="https://clickhouse.com/">ClickHouse</a>, <a href="https://www.influxdata.com/blog/influxdb-engine/">InfluxDB IOx</a>,
|
||
<a href="https://pinot.apache.org/">Apache Pinot</a>, and <a href="https://duckdb.org/">DuckDB</a> there are new Observability tools popping up all the time.</p>
|
||
<p>If you aren’t constrained, I <strong>highly recommend</strong> defaulting to using <a href="https://opentelemetry.io/">OpenTelemetry</a>
|
||
and <a href="https://www.honeycomb.io/">Honeycomb</a>. Your life will be easier.</p>
|
||
<p>However even if you are stuck in a corporate environment with a strong allergy to technology built after 2010 you
|
||
can leverage log search tools like ElasticSearch in a pinch. <a href="https://stripe.com/blog/canonical-log-lines">Stripe</a>’s
|
||
blog post goes over how to use Splunk for this.</p>
|
||
<p>In any tool you want to focus on getting proficient at 3 core techniques in order to sift through your events.
|
||
The faster you are able to apply these, iterate, and ask questions of your data, the better you’ll be able to
|
||
debug issues and see what your system is really doing. When observability folks refer to “slicing and dicing”
|
||
data, this is what they are generally referring to. I’ll represent queries using a made-up SQL dialect, but
|
||
you should be able to find equivalents in your tool’s query language.</p>
|
||
<h4 id="visualizing">Visualizing<a class="heading-anchor" aria-label="Link to this section" href="#visualizing">#</a></h4>
|
||
<p>Existing in a human body comes with its fair share of downsides, but the human visual cortex is really, really
|
||
good at recognizing patterns. Give it a fighting chance by getting really good at summoning visualizations
|
||
of the data your system is emitting. <code>COUNT</code>, <code>COUNT_DISTINCT</code>, <code>HEATMAP</code>, <code>P90</code>, <code>MAX</code>, <code>MIN</code>, Histogram.
|
||
Learn to leverage whatever graphs your tool makes available to you. Practice it. Get fast.</p>
|
||
<p><img src="/_astro/heatmaps.BtFRY6ZF_Z24Cffr.webp" alt="A Honeycomb screenshot of heatmap" loading="lazy" decoding="async" width="801" height="250"></p>
|
||
<p><img src="/_astro/splunk-histogram.DyaarE-V_Ukzwp.webp" alt="A Splunk screenshot of histogram" loading="lazy" decoding="async" width="3546" height="990"></p>
|
||
<h4 id="grouping">Grouping<a class="heading-anchor" aria-label="Link to this section" href="#grouping">#</a></h4>
|
||
<p>With each new annotation that we add to our wide events, we create another dimension along which we can
|
||
slice our data. <code>GROUP BY</code> allows us to look along that dimension and see if the values along that
|
||
dimension match our expectations.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> instance</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">id</span></span></code></pre>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> client</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">OS</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">client</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">version</span></span></code></pre>
|
||
<h4 id="filtering">Filtering<a class="heading-anchor" aria-label="Link to this section" href="#filtering">#</a></h4>
|
||
<p>Once we’ve narrowed in one dimension that is interesting, we usually want to dig further into
|
||
that data. Filtering down so that we’re only looking at data from one endpoint, or from one IP address,
|
||
or sent by the iOS app, or only from users with a specific feature flag turned on allows us to narrow our
|
||
focus to a very specific segment of traffic.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">WHERE</span><span style="color:#79B8FF"> http</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">route</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "/user/account"</span></span></code></pre>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">WHERE</span><span style="color:#79B8FF"> http</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">route</span><span style="color:#F97583"> !=</span><span style="color:#9ECBFF"> "/health"</span></span></code></pre>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">WHERE</span><span style="color:#79B8FF"> http</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">user_agent_header</span><span style="color:#E1E4E8"> contains </span><span style="color:#9ECBFF">"Android"</span></span></code></pre>
|
||
<h2 id="write-a-middleware-to-help-you">Write a middleware to help you<a class="heading-anchor" aria-label="Link to this section" href="#write-a-middleware-to-help-you">#</a></h2>
|
||
<p>If you are using an OpenTelemetry SDK it is already creating a wrapping span around the request and
|
||
response. You can access it by asking for the active span at any point during the processing of
|
||
the request.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="js"><code><span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> span </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> opentelemetry.trace.</span><span style="color:#B392F0">getActiveSpan</span><span style="color:#E1E4E8">();</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">span.</span><span style="color:#B392F0">setAttributes</span><span style="color:#E1E4E8">({</span></span>
|
||
<span class="line"><span style="color:#9ECBFF"> "user_agent.original"</span><span style="color:#E1E4E8">: c.req.</span><span style="color:#B392F0">header</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"User-Agent"</span><span style="color:#E1E4E8">),</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">});</span></span></code></pre>
|
||
<p>However if anyone wraps any of your code in a child span the “active span” will change to be that
|
||
new wrapping span! There is no first-class way of addressing this original “main” span in OpenTelemetry.
|
||
However, we can work around this by saving a reference to this specific span in the <a href="https://opentelemetry.io/docs/specs/otel/context/">context</a>
|
||
so we can always have access to the “main” wrapping span.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="js"><code><span class="line"><span style="color:#6A737D">// create a reference to store the span on the opentelemetry context object</span></span>
|
||
<span class="line"><span style="color:#F97583">const</span><span style="color:#79B8FF"> MAIN_SPAN_CONTEXT_KEY</span><span style="color:#F97583"> =</span><span style="color:#B392F0"> createContextKey</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"main_span_context_key"</span><span style="color:#E1E4E8">);</span></span>
|
||
<span class="line"></span>
|
||
<span class="line"><span style="color:#F97583">function</span><span style="color:#B392F0"> mainSpanMiddleware</span><span style="color:#E1E4E8">(</span><span style="color:#FFAB70">req</span><span style="color:#E1E4E8">, </span><span style="color:#FFAB70">res</span><span style="color:#E1E4E8">, </span><span style="color:#FFAB70">next</span><span style="color:#E1E4E8">) {</span></span>
|
||
<span class="line"><span style="color:#6A737D"> // pull the active span created by the http instrumentation</span></span>
|
||
<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> span </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> trace.</span><span style="color:#B392F0">getActiveSpan</span><span style="color:#E1E4E8">();</span></span>
|
||
<span class="line"></span>
|
||
<span class="line"><span style="color:#6A737D"> // get the current context</span></span>
|
||
<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> ctx </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> context.</span><span style="color:#B392F0">active</span><span style="color:#E1E4E8">();</span></span>
|
||
<span class="line"></span>
|
||
<span class="line"><span style="color:#6A737D"> // set any attributes we always want on the main span</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> span.</span><span style="color:#B392F0">setAttribute</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"main"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">true</span><span style="color:#E1E4E8">);</span></span>
|
||
<span class="line"></span>
|
||
<span class="line"><span style="color:#6A737D"> // OpenTelemetry context is immutable, so to modify it we create</span></span>
|
||
<span class="line"><span style="color:#6A737D"> // a new version with our span added</span></span>
|
||
<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> newCtx </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> ctx.</span><span style="color:#B392F0">setValue</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">MAIN_SPAN_CONTEXT_KEY</span><span style="color:#E1E4E8">, span);</span></span>
|
||
<span class="line"></span>
|
||
<span class="line"><span style="color:#6A737D"> // set that new context as active for the duration of the request</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> context.</span><span style="color:#B392F0">with</span><span style="color:#E1E4E8">(newCtx, () </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> {</span></span>
|
||
<span class="line"><span style="color:#B392F0"> next</span><span style="color:#E1E4E8">();</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> });</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">}</span></span>
|
||
<span class="line"></span>
|
||
<span class="line"><span style="color:#6A737D">// create another function that allows you to annotate this saved span easily</span></span>
|
||
<span class="line"><span style="color:#F97583">function</span><span style="color:#B392F0"> setMainSpanAttributes</span><span style="color:#E1E4E8">(</span><span style="color:#FFAB70">attributes</span><span style="color:#E1E4E8">) {</span></span>
|
||
<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> mainSpan </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> context.</span><span style="color:#B392F0">active</span><span style="color:#E1E4E8">().</span><span style="color:#B392F0">getValue</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">MAIN_SPAN_CONTEXT_KEY</span><span style="color:#E1E4E8">);</span></span>
|
||
<span class="line"><span style="color:#F97583"> if</span><span style="color:#E1E4E8"> (mainSpan) {</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> mainSpan.</span><span style="color:#B392F0">setAttributes</span><span style="color:#E1E4E8">(attributes);</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> }</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>
|
||
<p>Now our annotation code can look a little simpler, and we can always know that we’re setting these
|
||
attributes on the wrapping span.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="js"><code><span class="line"><span style="color:#B392F0">setMainSpanAttributes</span><span style="color:#E1E4E8">({</span></span>
|
||
<span class="line"><span style="color:#9ECBFF"> "user.id"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"123"</span><span style="color:#E1E4E8">,</span></span>
|
||
<span class="line"><span style="color:#9ECBFF"> "user.type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"enterprise"</span><span style="color:#E1E4E8">,</span></span>
|
||
<span class="line"><span style="color:#9ECBFF"> "user.auth_method"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"oauth"</span><span style="color:#E1E4E8">,</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">});</span></span></code></pre>
|
||
<p>You can play around with a minimal running example <a href="https://github.com/jmorrell/a-practitioners-guide-to-wide-events/tree/main/opentelemetry-js-example">here</a>.</p>
|
||
<p>At Heroku we had internal <a href="https://opentelemetry.io/docs/concepts/distributions/">OpenTelemetry Distributions</a>
|
||
that set this up for you automatically and added as many automatic annotations as possible to these spans.</p>
|
||
<p>If you are not using OpenTelemetry
|
||
<a href="https://gist.github.com/jmorrell/76a9ee631370e073d6e2616dc1f67feb">here’s a gist that might help you get started</a>.
|
||
<a href="https://jeremymorrell.dev/blog/minimal-js-tracing/">My previous post</a> may help you put this logic together.</p>
|
||
<h2 id="what-do-i-add-to-this-main-span">What do I add to this “main” span?<a class="heading-anchor" aria-label="Link to this section" href="#what-do-i-add-to-this-main-span">#</a></h2>
|
||
<figure class="social-card not-prose mx-auto my-8 max-w-xl rounded-2xl border border-stone-200 bg-white p-5 shadow-sm dark:border-stone-700 dark:bg-stone-800"><div class="flex items-start gap-3"><a href="https://x.com/mipsytipsy" target="_blank" rel="noopener noreferrer" aria-label="Charity Majors's profile" class="shrink-0"><img src="/_astro/mipsytipsy.D4oMMxdR_1hPjXY.webp" alt loading="lazy" decoding="async" width="96" height="96" class="size-12 rounded-full object-cover"></a><div class="min-w-0 flex-1"><a href="https://x.com/mipsytipsy" target="_blank" rel="noopener noreferrer" class="block truncate font-semibold text-black no-underline hover:underline dark:text-white">Charity Majors</a><a href="https://x.com/mipsytipsy" target="_blank" rel="noopener noreferrer" class="block truncate text-sm text-stone-500 no-underline hover:underline dark:text-stone-400">@mipsytipsy</a></div><a href="https://twitter.com/mipsytipsy/status/1744579558962336138" target="_blank" rel="noopener noreferrer" aria-label="View on X" title="View on X" class="shrink-0"><svg viewBox="0 0 24 24" width="20" height="20" class="fill-black dark:fill-white" aria-hidden="true"><path d="M18.244 2.25h3.308l-7.227 8.26 8.502 11.24H16.17l-5.214-6.817L4.99 21.75H1.68l7.73-8.835L1.254 2.25H8.08l4.713 6.231 5.451-6.231Zm-1.161 17.52h1.833L7.084 4.126H5.117l11.966 15.644Z"></path></svg></a></div><blockquote class="mt-3 whitespace-pre-line text-[0.95rem] leading-relaxed text-black/85 dark:text-white/85">And how many dimensions do you plan to emit and pack into your wide events?
|
||
|
||
MANY. Hundreds! The more you have, the better you can detect and correlate rare conditions with precision.
|
||
|
||
As you adjust to the joys of debugging with rich context, you will itch for it everywhere. ☺️</blockquote><figcaption class="mt-3 text-sm"><a href="https://twitter.com/mipsytipsy/status/1744579558962336138" target="_blank" rel="noopener noreferrer" class="text-stone-500 no-underline hover:underline dark:text-stone-400">January 9, 2024</a></figcaption></figure>
|
||
<p>We need to add attributes about the request, and there are likely far more of these than you would expect.
|
||
It’s easy to come up with a dozen or so, but in a well-instrumented code base there will be hundreds of attributes.</p>
|
||
<p>Note that while this is a long list, it is definitely not exhaustive. OpenTelemetry defines sets of attribute names as
|
||
<a href="https://opentelemetry.io/docs/specs/semconv/">Semantic Conventions</a> that can also be used for inspiration. I have tried
|
||
to follow these in my naming where possible.</p>
|
||
<h3 id="a-convention-to-filter-out-everything-else">A convention to filter out everything else<a class="heading-anchor" aria-label="Link to this section" href="#a-convention-to-filter-out-everything-else">#</a></h3>
|
||
<p>Traces contain lots of spans, so it’s helpful to have a convention for identifying and searching for these “wide events”.
|
||
<code>root</code> and <code>canon</code> were floated as options, but I’ve landed on calling them <code>main</code> spans.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>main</code></td><td><code>true</code></td><td>Present only for spans designated as a “wide event”, usually wrapping a request / response, or a background job</td></tr></tbody></table></div>
|
||
<p>This convention allows you to quickly figure out “what does the traffic to this service look like?” with a single query:</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> COUNT</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">*</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> http</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">route</span></span></code></pre>
|
||
<p><img src="/_astro/traffic-by-route.Bvy3b1dc_ZPgldd.webp" alt="Graph of traffic grouped by route over a week. There is an anomally." loading="lazy" decoding="async" width="1227" height="691"></p>
|
||
<h3 id="service-metadata">Service metadata<a class="heading-anchor" aria-label="Link to this section" href="#service-metadata">#</a></h3>
|
||
<p>Of course we need to add some information about the service we’re running. Consider adding additional metadata about
|
||
which team owns the system, or which Slack channel the owning team hangs out in, though note that this can be
|
||
tedious to update if your workplace experiences frequent re-orgs. Tying these to a service catalog like <a href="https://backstage.io/">Backstage</a>
|
||
is left as an exercise to the reader.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>service.name</code></td><td><code>api</code> <br/> <code>shoppingcart</code></td><td>What is the name of this service?</td></tr><tr><td><code>service.environment</code></td><td><code>production</code> <br/> <code>staging</code> <br/> <code>development</code></td><td>Where is this service running?</td></tr><tr><td><code>service.team</code></td><td><code>web-services</code> <br/> <code>dev-ex</code></td><td>Which team owns this service. Useful for knowing who to page in during incidents.</td></tr><tr><td><code>service.slack_channel</code></td><td><code>web-services</code> <br/> <code>dev-ex</code></td><td>If I discover an issue with this service, where should I reach out?</td></tr></tbody></table></div>
|
||
<blockquote>
|
||
<p>How many services does each team run?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> COUNT_DISTINCT(</span><span style="color:#79B8FF">service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">environment</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "production"</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">team</span></span></code></pre>
|
||
<p>Ever look at the load on a system and then wonder “Is that appropriate for the machine this is running on?”, and
|
||
now you have to look through other tools or config files to get that information. Throw that context on the wide
|
||
event so that it’s available when you need it.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>instance.id</code></td><td><code>656993bd-40e1-4c76-baff-0e50e158c6eb</code></td><td>An ID that maps to this one instance of the service</td></tr><tr><td><code>instance.memory_mb</code></td><td><code>12336</code></td><td>How much RAM is available to this service?</td></tr><tr><td><code>instance.cpu_count</code></td><td><code>4</code> <br/> <code>8</code> <br/> <code>196</code></td><td>How many cores are available to this service?</td></tr><tr><td><code>instance.type</code></td><td><code>m6i.xlarge</code></td><td>Does your vendor have a name for this type of instance?</td></tr></tbody></table></div>
|
||
<blockquote>
|
||
<p>What are the services with the most memory that we run? What instance types do they use?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#E1E4E8">,</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> instance</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">memory_mb</span><span style="color:#E1E4E8">,</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> instance</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">type</span></span>
|
||
<span class="line"><span style="color:#F97583">ORDER BY</span><span style="color:#79B8FF"> instance</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">memory_mb</span><span style="color:#F97583"> DESC</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">instance</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">type</span></span>
|
||
<span class="line"><span style="color:#F97583">LIMIT</span><span style="color:#79B8FF"> 10</span></span></code></pre>
|
||
<p>However you’re orchestrating your systems make sure that all of the relevant information is added. I’ve included some
|
||
examples from <a href="https://opentelemetry.io/docs/specs/semconv/resource/k8s/">the Kubernetes semantic conventions</a> for inspiration.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>container.id</code></td><td><code>a3bf90e006b2</code></td><td>An ID used to identify Docker containers</td></tr><tr><td><code>container.name</code></td><td><code>nginx-proxy</code> <br/> <code>wordpress-app</code></td><td>Container name used by container runtime</td></tr><tr><td><code>k8s.cluster.name</code></td><td><code>api-cluster</code></td><td>Name of the kubernetes cluster your service is running in</td></tr><tr><td><code>k8s.pod.name</code></td><td><code>nginx-2723453542-065rx</code></td><td>Name of the kubernetes pod your service is running in</td></tr><tr><td><code>cloud.availability_zone</code></td><td><code>us-east-1c</code></td><td>AZ where you’re running your service</td></tr><tr><td><code>cloud.region</code></td><td><code>us-east-1</code></td><td>Region where you’re running your service</td></tr></tbody></table></div>
|
||
<p>But even if you’re using a Platform-as-a-Service you can still pull out a lot of useful information!</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>heroku.dyno</code></td><td><code>web.1</code> <br/> <code>worker.3</code></td><td>The env var <code>DYNO</code> that is set on your app at runtime</td></tr><tr><td><code>heroku.dyno_type</code></td><td><code>web</code> <br/> <code>worker</code></td><td>The first part of the <code>DYNO</code> env var before the <code>.</code>. Separating this makes it easier to query</td></tr><tr><td><code>heroku.dyno_index</code></td><td><code>1</code> <br/> <code>3</code></td><td>The second part of the <code>DYNO</code> env var after the <code>.</code>. Separating this makes it easier to query</td></tr><tr><td><code>heroku.dyno_size</code></td><td><code>performance-m</code></td><td>The selected dyno size</td></tr><tr><td><code>heroku.space</code></td><td><code>my-private-space</code></td><td>The name of the private space that your are deployed into</td></tr><tr><td><code>heroku.region</code></td><td><code>virginia</code> <br/> <code>oregon</code></td><td>Which region is this app located in?</td></tr></tbody></table></div>
|
||
<blockquote>
|
||
<p>How many dynos are we running? What dyno types are they? For which services?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> COUNT_DISTINCT(</span><span style="color:#79B8FF">heroku</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">dyno_index</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">heroku</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">dyno_type</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">instance</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">type</span></span></code></pre>
|
||
<h3 id="build-info">Build info<a class="heading-anchor" aria-label="Link to this section" href="#build-info">#</a></h3>
|
||
<p>Inevitably some of the first questions asked in any incident are “Did something just go out?” or “What changed?”.
|
||
Instead of jumping to your deployment tool or looking through GitHub repositories, add that data to your telemetry.</p>
|
||
<p>Threading this data from your build system through to your production system so that it’s available at runtime can
|
||
be a non-trivial amount of glue code, but having this information easily available during incidents is invaluable.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>service.version</code></td><td><code>v123</code> <br/> <code>9731945429d3d083eb78666c565c61bcef39a48f</code></td><td>However you track your version, ex: a version string or a hash of the built image</td></tr><tr><td><code>service.build.id</code></td><td><code>acd8bb57-fb9f-4b2d-a750-4315e99dac64</code></td><td>If your build system gives you an ID, this context allows you to audit the build if something goes wrong</td></tr><tr><td><code>service.build.git_hash</code></td><td><code>6f6466b0e693470729b669f3745358df29f97e8d</code></td><td>The git SHA of the deployed commit so you can know exactly which code was running</td></tr><tr><td><code>service.build.pull_request_url</code></td><td><code>https://github.com/your-company/api-service/pull/121</code></td><td>The url of the pull request that was merged that triggered the deploy</td></tr><tr><td><code>service.build.diff_url</code></td><td><code>https://github.com/your-company/api-service/compare/c9d9380..05e5736</code></td><td>A url that compares the previously deployed commit against the newly deployed commit</td></tr><tr><td><code>service.build.deployment.at</code></td><td><code>2024-10-14T19:47:38Z</code></td><td>Timestamp when the deployment process started</td></tr><tr><td><code>service.build.deployment.user</code></td><td><code>keanu.reeves@your-company.com</code></td><td>Which authenticated user kicked off the build? Could be a bot</td></tr><tr><td><code>service.build.deployment.trigger</code></td><td><code>merge-to-main</code> <br/> <code>slack-bot</code> <br/> <code>api-request</code> <br/> <code>config-change</code></td><td>What triggered the deploy? Extremely valuable context during an deploy-triggered incident</td></tr><tr><td><code>service.build.deployment.age_minutes</code></td><td><code>1</code> <br/> <code>10230</code></td><td>How old is this deploy? Shortcuts the frequent incident question “Did something just go out?”</td></tr></tbody></table></div>
|
||
<p><strong>Won’t this be a lot of repetitive data?</strong> These values do not change except between deploys! See <a href="#frequent-objections">Frequent Objections</a></p>
|
||
<blockquote>
|
||
<p>What systems have recently been deployed?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#E1E4E8">,</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> MIN</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">build</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">deployment</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">age_minutes</span><span style="color:#E1E4E8">) </span><span style="color:#F97583">as</span><span style="color:#E1E4E8"> age</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">build</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">deployment</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">age_minutes</span><span style="color:#F97583"> <</span><span style="color:#79B8FF"> 20</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span></span>
|
||
<span class="line"><span style="color:#F97583">ORDER BY</span><span style="color:#E1E4E8"> age </span><span style="color:#F97583">ASC</span></span>
|
||
<span class="line"><span style="color:#F97583">LIMIT</span><span style="color:#79B8FF"> 10</span></span></code></pre>
|
||
<blockquote>
|
||
<p>What’s up with the spike of 500s when we did the last deploy?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> COUNT</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">*</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span><span style="color:#F97583"> AND</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> http</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">status_code</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">version</span></span></code></pre>
|
||
<p><img src="/_astro/group-by-http-and-status.tDSRPVAz_Z17h9S0.webp" alt="Graph showing requests grouped by http status code and version. There is a spike of 500s correlating to v1 shutting down." loading="lazy" decoding="async" width="1616" height="926"></p>
|
||
<h3 id="http">HTTP<a class="heading-anchor" aria-label="Link to this section" href="#http">#</a></h3>
|
||
<p>You should get most of these from your tracing library instrumentation, but there are usually more you can add if, for example,
|
||
your organization uses non-standard headers. Don’t settle for only what OpenTelemetry gives you by default!</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>server.address</code></td><td><code>example.com</code> <br/> <code>localhost</code></td><td>Name of the HTTP server that received the request</td></tr><tr><td><code>url.path</code></td><td><code>/checkout</code> <br/> <code>/account/123/features</code></td><td>URI path after the domain</td></tr><tr><td><code>url.scheme</code></td><td><code>http</code>, <code>https</code></td><td>URI scheme</td></tr><tr><td><code>url.query</code></td><td><code>q=test</code>, <code>ref=####</code></td><td>URI query component</td></tr><tr><td><code>http.request.id</code></td><td><code>79104EXAMPLEB723</code></td><td>Platform request id: ex: <code>x-request-id</code>, <code>x-amz-request-id</code></td></tr><tr><td><code>http.request.method</code></td><td><code>GET</code> <br/> <code>PUT</code> <br/> <code>POST</code> <br/> <code>OPTIONS</code></td><td>HTTP request method</td></tr><tr><td><code>http.request.body_size</code></td><td><code>3495</code></td><td>Size of the request payload body in bytes</td></tr><tr><td><code>http.request.header.content-type</code></td><td><code>application/json</code></td><td>Value of a specific request header, “content-type” in this case, but there are many more. Pick out any that are important for your service</td></tr><tr><td><code>http.response.status_code</code></td><td><code>200</code> <br/> <code>404</code> <br/> <code>500</code></td><td>HTTP response status code</td></tr><tr><td><code>http.response.body_size</code></td><td><code>1284</code> <br/> <code>2202009</code></td><td>Size of the response payload body in bytes</td></tr><tr><td><code>http.request.header.content-type</code></td><td><code>text/html</code></td><td>Value of a specific response header, “content-type” in this case, but there are many more. Pick out any that are important for your service</td></tr></tbody></table></div>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> HEATMAP(</span><span style="color:#79B8FF">http</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">response</span><span style="color:#E1E4E8">.body_size),</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span></span></code></pre>
|
||
<p><img src="/_astro/http-body-size-annotated.BXCOh_iU_ZxO0NW.webp" alt="A heatmap of response sizes. Most are within a fixed band, but there are sharp outliers that warrant more investigation." loading="lazy" decoding="async" width="1769" height="1027"></p>
|
||
<p><code>User-Agent</code> headers contain a wealth of info. Don’t rely on regex queries to try and make sense of them down the road. Parse them
|
||
into structured data from the beginning.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>user_agent.original</code></td><td><code>Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.3</code></td><td>The value of the HTTP <code>User-Agent</code> header</td></tr><tr><td><code>user_agent.device</code></td><td><code>computer</code> <br/> <code>tablet</code> <br/> <code>phone</code></td><td>Device type derived from the <code>User-Agent</code> header</td></tr><tr><td><code>user_agent.OS</code></td><td><code>Windows</code> <br/> <code>MacOS</code></td><td>OS derived from the <code>User-Agent</code> header</td></tr><tr><td><code>user_agent.browser</code></td><td><code>Chrome</code> <br/> <code>Safari</code> <br/> <code>Firefox</code></td><td>Browser derived from the <code>User-Agent</code> header</td></tr><tr><td><code>user_agent.browser_version</code></td><td><code>129</code> <br/> <code>18.0</code></td><td>Browser version derived from the <code>User-Agent</code> header</td></tr></tbody></table></div>
|
||
<blockquote>
|
||
<p>What browsers are my users using?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> COUNT</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">*</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> user_agent</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">browser</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">user_agent</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">browser_version</span></span></code></pre>
|
||
<p>If you have any custom user agents or headers used as a convention within your org parse that out too.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>user_agent.service</code></td><td><code>api-gateway</code> <br/> <code>auth-service</code></td><td>If you have a distributed architecture, have each service send a custom <code>User-Agent</code> header with its name and version</td></tr><tr><td><code>user_agent.service_version</code></td><td><code>v123</code> <br/> <code>6f6466b0e693470729b669f3745358df29f97e8d</code></td><td>If you have a distributed architecture, have each service send a custom <code>User-Agent</code> header with its name and version</td></tr><tr><td><code>user_agent.app</code></td><td><code>iOS</code> <br/> <code>android</code></td><td>If a request is coming from a mobile app, make sure it includes which app and its version</td></tr><tr><td><code>user_agent.app_version</code></td><td><code>v123</code> <br/> <code>6f6466b0e693470729b669f3745358df29f97e8d</code></td><td>If a request is coming from a mobile app, make sure it includes which app and its version</td></tr></tbody></table></div>
|
||
<h3 id="route-info">Route info<a class="heading-anchor" aria-label="Link to this section" href="#route-info">#</a></h3>
|
||
<p>We’re not done with HTTP attributes yet! One of the most important bits is the API endpoint
|
||
that the request matched. OpenTelemetry SDKs will <em>usually</em> give this to you automagically
|
||
but not always. Consider extracting the route parameters and query parameters as additional attributes.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>http.route</code></td><td><code>/team/{team_id}/user/{user_id}</code></td><td>The route pattern that the url path is matched against</td></tr><tr><td><code>http.route.param.team_id</code></td><td><code>14739</code> <br/> <code>team-name-slug</code></td><td>The extracted segment of the url path as it is parsed for each parameter</td></tr><tr><td><code>http.route.query.sort_dir</code></td><td><code>asc</code></td><td>The query parameters that are relevant to the response of your service. Ex: <code>?sort_dir=asc&...</code></td></tr></tbody></table></div>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> P99(duration_ms)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> http</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">route</span></span></code></pre>
|
||
<p><img src="/_astro/p99-duration-annotated.BXlgufk3_1hdF1S.webp" alt="A chart of p99's broken down by route. There is a spike on only some of them. We should break down by version now to check if this was caused by a deploy." loading="lazy" decoding="async" width="1769" height="1012"></p>
|
||
<h3 id="user-and-customer-info">User and customer info<a class="heading-anchor" aria-label="Link to this section" href="#user-and-customer-info">#</a></h3>
|
||
<p>Once you get the basics down, this is <strong>the most important</strong> piece of metadata that you can add. No automagic SDK
|
||
will be able to encode the particulars of your user model.</p>
|
||
<p>It’s common for a single user or account to be responsible for a 10%+ of a business’ revenue, and frequently their
|
||
usage patterns look significantly different than the average user. They probably have more users, store more data,
|
||
and hit limits and edge-cases that will never show up for the user paying $10 / month. Be sure you can separate
|
||
their traffic from others.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>user.id</code></td><td><code>2147483647</code> <br/> <code>user@example.com</code></td><td>The primary ID for a user. If this is an email and you’re using a vendor, consider your org’s policy on putting PII in external services.</td></tr><tr><td><code>user.type</code></td><td><code>free</code> <br/> <code>premium</code> <br/> <code>enterprise</code> <br/> <code>vip</code></td><td>How does the business see this type of user? Individual accounts are sometimes responsible for 10%+ of a business’ income. Make sure you can separate their traffic from others!</td></tr><tr><td><code>user.auth_method</code></td><td><code>token</code> <br/> <code>basic-auth</code> <br/> <code>jwt</code> <br/> <code>sso-github</code></td><td>How did this user authenticate into your system?</td></tr><tr><td><code>user.team.id</code></td><td><code>5387</code> <br/> <code>web-services</code></td><td>If you have a team construct, which one does this user belong to?</td></tr><tr><td><code>user.org.id</code></td><td><code>278</code> <br/> <code>enterprise-name</code></td><td>If this user is part of an organization with an enterprise contract, track that!</td></tr><tr><td><code>user.age_days</code></td><td><code>0</code> <br/> <code>637</code></td><td>Not the user’s literal age, but how long ago was this account created? Is this an issue experienced by someone new to your app, or only once they’ve saved a lot of data?</td></tr><tr><td><code>user.assumed</code></td><td><code>true</code></td><td>Have an internal way of assuming a user’s identity for debugging? Be sure to track this</td></tr><tr><td><code>user.assumed_by</code></td><td><code>engineer-3@your-company.com</code></td><td>And track which actual user is assuming the user’s identity</td></tr></tbody></table></div>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> P99(duration_ms)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> user</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">type</span></span></code></pre>
|
||
<h3 id="rate-limits">Rate limits<a class="heading-anchor" aria-label="Link to this section" href="#rate-limits">#</a></h3>
|
||
<p>Whatever your rate limiting strategy, make sure the current rate limit info gets added too. Can you quickly find
|
||
examples of users that are being rate-limited by your service?</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>ratelimit.limit</code></td><td><code>200000</code></td><td>You might not now, but you will likely have users with different rate limits in the future, note down what the actual limit is for this request</td></tr><tr><td><code>ratelimit.remaining</code></td><td><code>130000</code></td><td>What is the budget remaining for this user?</td></tr><tr><td><code>ratelimit.used</code></td><td><code>70000</code></td><td>How many requests have been used in the current rate window</td></tr><tr><td><code>ratelimit.reset_at</code></td><td><code>2024-10-14T19:47:38Z</code></td><td>When will the rate limit be reset next? if applicable</td></tr></tbody></table></div>
|
||
<blockquote>
|
||
<p>This user has a support ticket open about being rate-limited. Let’s see what they were doing</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> COUNT</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">*</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span><span style="color:#F97583"> AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> user</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">id</span><span style="color:#F97583"> =</span><span style="color:#79B8FF"> 5838</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> http</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">route</span></span></code></pre>
|
||
<p><img src="/_astro/rate-limit-investigation-annotated.DGJPVuPW_e2lXY.webp" alt="A graph of one users activity. There is a big spike hitting the same route a lot at the end. This gives us a starting point for investigation" loading="lazy" decoding="async" width="1769" height="1012"></p>
|
||
<blockquote>
|
||
<p>What routes are users who have burned most of their rate limit hitting? Does this activity look suspicious?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> COUNT</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">*</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span><span style="color:#F97583"> AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> ratelimit</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">remaining</span><span style="color:#F97583"> <</span><span style="color:#79B8FF"> 100</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> http</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">route</span></span></code></pre>
|
||
<h3 id="caching">Caching<a class="heading-anchor" aria-label="Link to this section" href="#caching">#</a></h3>
|
||
<p>For every code path where we could shortcut with a cache response, add whether or not it was successful</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>cache.session_info</code></td><td><code>true</code> <br/> <code>false</code></td><td>Was the session info cached or did it need to be re-fetched?</td></tr><tr><td><code>cache.feature_flags</code></td><td><code>true</code> <br/> <code>false</code></td><td>Were the feature flags cached for this user or did they need to be re-fetched?</td></tr></tbody></table></div>
|
||
<h3 id="localization-info">Localization info<a class="heading-anchor" aria-label="Link to this section" href="#localization-info">#</a></h3>
|
||
<p>What localization options has the user chosen? This can be a frequent source of bugs</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>localization.language_dir</code></td><td><code>rtl</code>, <code>ltr</code></td><td>Which direction is text laid out in their language?</td></tr><tr><td><code>localization.country</code></td><td><code>mexico</code>, <code>uk</code></td><td>Which country are they from?</td></tr><tr><td><code>localization.currency</code></td><td><code>USD</code>, <code>CAD</code></td><td>Which currency have they chosen to work with?</td></tr></tbody></table></div>
|
||
<h3 id="uptime">Uptime<a class="heading-anchor" aria-label="Link to this section" href="#uptime">#</a></h3>
|
||
<p>Tracking how long the service has been running when it serves a request can help you visualize several classes of bugs:</p>
|
||
<ul>
|
||
<li>Issues that show up on a reboot</li>
|
||
<li>Memory leaks that only start to show up when the service has been running for a long time</li>
|
||
<li>Frequent crashes / restarts if you have automatically restart the service on failure</li>
|
||
</ul>
|
||
<p>I recommend also either adding the <code>log10</code> of the uptime or having some way of visualizing this. When graphed this emphasizes
|
||
the important first few minutes of a service without being squished into the bottom of the graph by instances with several days
|
||
or more of uptime.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>uptime_sec</code></td><td><code>1533</code></td><td>How long has this instance of your app been running? Can be useful to visualize to see restarts</td></tr><tr><td><code>uptime_sec_log_10</code></td><td><code>3.185</code></td><td>Grows sub-linearly which allows you to visualize long-running services and brand new ones on the same graph</td></tr></tbody></table></div>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> HEATMAP(uptime_sec),</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> HEATMAP(uptime_sec_log_10)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span></span></code></pre>
|
||
<p><img src="/_astro/uptime.CJbFmMOT_qrIok.webp" alt="Heatmaps of uptime when a service enters a crash loop. It's far easier to distinguish in log scale" loading="lazy" decoding="async" width="1810" height="1895"></p>
|
||
<h3 id="metrics">Metrics<a class="heading-anchor" aria-label="Link to this section" href="#metrics">#</a></h3>
|
||
<p>This one might be a bit controversial, but I’ve found it helpful to tag spans with context about what the system
|
||
was experiencing while processing the request. We fetch this information every ~10 seconds, cache it, and add it
|
||
to every main span produced during that time.</p>
|
||
<p>Capturing metrics in this way is not mathematically sound. Since you only get data when traffic is flowing, you
|
||
can’t calculate a <code>P90</code> for cpu load that would stand up to any rigorous scrutiny, but that’s actually fine
|
||
in practice. It’s close enough to get some quick signal while you’re debugging without switching to a different tool,
|
||
especially if you can avoid calculations and visualize with a heatmap.</p>
|
||
<p>I wouldn’t recommend setting alerts on this data though. Plain ol’ metrics are great for that.</p>
|
||
<p><a href="https://jessitron.com/">Jessica Kerr</a> recently wrote about this approach on the <a href="https://www.honeycomb.io/blog/get-infinite-custom-metrics-for-free">Honeycomb Blog</a>.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>metrics.memory_mb</code></td><td><code>153</code> <br/> <code>2593</code></td><td>How much memory is being used by the system at the time its service this request</td></tr><tr><td><code>metrics.cpu_load</code></td><td><code>0.57</code> <br/> <code>5.89</code></td><td>CPU load of the system service this request. Given as # of active cores</td></tr><tr><td><code>metrics.gc_count</code></td><td><code>5390</code></td><td>Last observed number of garbage collections. Could be cumulative (total since service started) or delta (ex: number in the last minute)</td></tr><tr><td><code>metrics.gc_pause_time_ms</code></td><td><code>14</code> <br/> <code>325</code></td><td>Time spent in garbage collections. Could also be cumulative or delta. Pick one and document which</td></tr><tr><td><code>metrics.go_routines_count</code></td><td><code>3</code> <br/> <code>3000</code></td><td>Number of go routines running</td></tr><tr><td><code>metrics.event_loop_latency_ms</code></td><td><code>0</code> <br/> <code>340</code></td><td>Cumulative time spent waiting on the next event loop tick. An important metric for Node apps</td></tr></tbody></table></div>
|
||
<blockquote>
|
||
<p>Are these requests getting slow because we’re running out of memory or CPU?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> HEATMAP(duration_ms),</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> HEATMAP(</span><span style="color:#79B8FF">metrics</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">memory_mb</span><span style="color:#E1E4E8">),</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> HEATMAP(</span><span style="color:#79B8FF">metrics</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">cpu_load</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> instance</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">id</span></span></code></pre>
|
||
<p><img src="/_astro/metrics.BHs4tp2f_Z1IJUPM.webp" alt="An example showing using the metrics data tagged on the span to get context for whats happening with the system" loading="lazy" decoding="async" width="2982" height="4630"></p>
|
||
<h3 id="async-request-summaries">Async request summaries<a class="heading-anchor" aria-label="Link to this section" href="#async-request-summaries">#</a></h3>
|
||
<p>When using a tracing system async requests should get their own spans, but it can still be useful to roll up
|
||
some stats to identify outliers and quickly find interesting traces.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>stats.http_requests_count</code></td><td><code>1</code> <br/> <code>140</code></td><td>How many http requests were triggered during the processing of this request?</td></tr><tr><td><code>stats.http_requests_duration_ms</code></td><td><code>849</code></td><td>Cumulative time spent in these http requests</td></tr><tr><td><code>stats.postgres_query_count</code></td><td><code>7</code> <br/> <code>742</code></td><td>How many Postgres queries were triggered during the processing of this request?</td></tr><tr><td><code>stats.postgres_query_duration_ms</code></td><td><code>1254</code></td><td>Cumulative time spent in these Postgres queries</td></tr><tr><td><code>stats.redis_query_count</code></td><td><code>3</code> <br/> <code>240</code></td><td>How many redis queries were triggered during the processing of this request?</td></tr><tr><td><code>stats.redis_query_duration_ms</code></td><td><code>43</code></td><td>Cumulative time spent in these redis queries</td></tr><tr><td><code>stats.twilio_calls_count</code></td><td><code>1</code> <br/> <code>4</code></td><td>How many calls to this vendors api were triggered during the processing of this request?</td></tr><tr><td><code>stats.twilio_calls_duration_ms</code></td><td><code>2153</code></td><td>Cumulative time spent in these vendor calls</td></tr></tbody></table></div>
|
||
<blockquote>
|
||
<p>Surely my service makes a reasonable number of calls to the database… right?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> HEATMAP(</span><span style="color:#79B8FF">stats</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">postgres_query_count</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span></span></code></pre>
|
||
<p><img src="/_astro/postgres-queries.BANqR6rQ_Vuns8.webp" alt="A heatmap of db queries per request. There is a bi-modal distribution but also some outliers that make a lot of requests" loading="lazy" decoding="async" width="1284" height="700"></p>
|
||
<p><strong>Instead of adding this explicitly, couldn’t we aggregate this by querying the whole trace?</strong> See <a href="#frequent-objections">Frequent Objections</a></p>
|
||
<h3 id="sampling">Sampling<a class="heading-anchor" aria-label="Link to this section" href="#sampling">#</a></h3>
|
||
<p>Once you start collecting fine-grained telemetry from your systems at a significant scale you run head-on
|
||
into the problem of sampling. Running systems can produce a lot of data! Engineers frequently want to store
|
||
and query all of it. Exact answers always! Make it fast! Also cheap! But it’s trade-offs all the way down.
|
||
Telemetry data is fundamentally different from the transaction data you’re storing for your users, and you
|
||
should think about it differently.</p>
|
||
<p>Luckily you only really need a statistically significant subset of the full dataset. Even sampling 1 out of every
|
||
1000 requests can provide a suprisingly detailed picture of the overall traffic patterns in a system.</p>
|
||
<p>Sampling is a suprisingly deep topic. Keep it simple if you’re starting and do uniform random head sampling,
|
||
but track your sample rate per-span so you can be ready for more sophisticated approaches down-the-line.</p>
|
||
<p>Good tooling will weight your calculations with a per-span, so you don’t have to mentally multiple the <code>COUNT</code>
|
||
call by the <code>sample_rate</code> to get an accurate answer. Here are some relevant articles:</p>
|
||
<ul>
|
||
<li><a href="https://research.facebook.com/file/2964294030497318/scuba-diving-into-data-at-facebook.pdf">I was first introduced to this idea in the Scuba paper</a></li>
|
||
<li><a href="https://docs.honeycomb.io/manage-data-volume/sample/sampled-data-in-honeycomb/">Honeycomb supports per-event sample rates</a></li>
|
||
<li><a href="https://blog.cloudflare.com/explaining-cloudflares-abr-analytics/">Cloudflare’s Analytics Engine will automatically sample for you based on volume</a></li>
|
||
</ul>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>sample_rate</code></td><td><code>1</code> <br/> <code>500</code></td><td><code>N</code> where 1 in <code>N</code> events will be sampled and stored and the rest dropped. If you’re sampling <code>1%</code> of requests, the <code>sample_rate</code> would be <code>100</code></td></tr></tbody></table></div>
|
||
<h3 id="timings">Timings<a class="heading-anchor" aria-label="Link to this section" href="#timings">#</a></h3>
|
||
<p>I find it super useful to break up the work that gets done to respond to a request into a handful of important chunks
|
||
and track how long each segment took on the main span.</p>
|
||
<blockquote>
|
||
<p>Wait, isn’t that what child spans are for?</p>
|
||
</blockquote>
|
||
<p>Wrapping absolutely everything in its own span is the most common failure mode I see when engineers first get access
|
||
to tracing tools. You have to design the structure of your data for the way you want to query it.</p>
|
||
<p>Child spans are helpful for waterfall visualization for a single request, but can be difficult to query and visualize
|
||
across <em>all</em> of your requests. Putting that information on a single span makes it easier to query and also helps with
|
||
tools like <a href="https://www.honeycomb.io/bubbleup">Honeycomb’s BubbleUp</a> which can then immediately tell you that
|
||
that group of requests was slow because authentication took 10 seconds for some reason.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>auth.duration_ms</code></td><td><code>52.2</code> <br/> <code>0.2</code></td><td>How long did we spend performing authentication during this request?</td></tr><tr><td><code>payload_parse.duration_ms</code></td><td><code>22.1</code> <br/> <code>0.1</code></td><td>Identify the core workloads of the service and add timings for them</td></tr></tbody></table></div>
|
||
<h3 id="errors">Errors<a class="heading-anchor" aria-label="Link to this section" href="#errors">#</a></h3>
|
||
<p>If you encounter an error and need to fail the operation, tag the span with the error information: type, stacktrace, etc.</p>
|
||
<p>One approach that I have found super-valuable is tagging each location where we throw an error with a unique slug describing
|
||
the error. If this string is unique within your codebase, it is easily found with a quick search. This allows someone
|
||
to jump straight from a spike in errors on a dashboard to the exact line of code that throwing the error. It also provides
|
||
a convenient low-cardinality field to <code>GROUP BY</code>.</p>
|
||
<p>You’re unlikely to be able to wrap all possible errors, but any time a failed request doesn’t have an <code>exception.slug</code>
|
||
that is a good sign that you have places in your code where your error handling could be improved. It’s now really easy
|
||
to find examples of requests that failed in ways you didn’t anticipate.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="js"><code><span class="line"><span style="color:#F97583">if</span><span style="color:#B392F0"> isNotRecoverable</span><span style="color:#E1E4E8">(err) {</span></span>
|
||
<span class="line"><span style="color:#6A737D"> // note the use of a plain string, not a variable, not dynamically generated</span></span>
|
||
<span class="line"><span style="color:#6A737D"> // consider enforcing this with custom lint rules</span></span>
|
||
<span class="line"><span style="color:#B392F0"> setErrorAttributes</span><span style="color:#E1E4E8">(err, </span><span style="color:#9ECBFF">"err-stripe-call-failed-exhausted-retries"</span><span style="color:#E1E4E8">);</span></span>
|
||
<span class="line"><span style="color:#F97583"> throw</span><span style="color:#E1E4E8"> err;</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>error</code></td><td><code>true</code> <br/> <code>false</code></td><td>Special field for whether the request failed or not</td></tr><tr><td><code>exception.message</code></td><td><code>Can't convert 'int' object to str</code> <br/> <code>undefined is not a function</code></td><td>The exception message encoded in the exception</td></tr><tr><td><code>exception.type</code></td><td><code>IOError</code> <br/> <code>java.net.ConnectException</code></td><td>The programmatic type of the exception</td></tr><tr><td><code>exception.stacktrace</code></td><td><code>ReferenceError: user is not defined</code><br/><code>at myFunction (/path/to/file.js:12:2)</code><br/><code>...</code></td><td>Capture the stack trace if its available to help pin-point where the error is being thrown</td></tr><tr><td><code>exception.expected</code></td><td><code>true</code>, <code>false</code></td><td>Is this an expected exception like a bot trying to hit a url that doesn’t exist? Allows filtering out of exceptions we can’t prevent but don’t need to worry about</td></tr><tr><td><code>exception.slug</code></td><td><code>auth-error</code> <br/> <code>invalid-route</code> <br/> <code>github-api-unavailable</code></td><td>Create a unique grepp-able slug-value to identify the code location of an error if its predictable during development time</td></tr></tbody></table></div>
|
||
<blockquote>
|
||
<p>Which of our enterprise users hit the most errors last week? And which one?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> COUNT_DISTINCT(</span><span style="color:#79B8FF">user</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">id</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span><span style="color:#F97583"> AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> user</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">type</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "enterprise"</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> exception</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">slug</span></span></code></pre>
|
||
<blockquote>
|
||
<p>Show me traces where we likely need to improve our error handling</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> trace</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">trace_id</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span><span style="color:#F97583"> AND</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> error </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> exception</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">slug</span><span style="color:#F97583"> =</span><span style="color:#F97583"> NULL</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> trace</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">trace_id</span></span></code></pre>
|
||
<h3 id="feature-flags">Feature flags<a class="heading-anchor" aria-label="Link to this section" href="#feature-flags">#</a></h3>
|
||
<p>Fine-grained feature flags are a developer super power that allows you to test code changes in production with only a fraction of
|
||
your users or traffic. Adding the flag information per-request allows you to compare how the new code is working as you opt more
|
||
of your traffic into the new code path. Coupled with the broad visibility you can get with wide events, and this can make even tricky
|
||
migrations vastly more manageable and allow you to ship code with confidence.</p>
|
||
<p>Note that semantic conventions differ here and suggest adding feature flag information as
|
||
<a href="https://opentelemetry.io/docs/specs/semconv/feature-flags/feature-flags-spans/">events on the span</a>.
|
||
I would suggest following that standard since it will ultimately have the best support from vendors if it’s moved to stable,
|
||
but especially in the mean time, I’m also putting this info on the main span.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>feature_flag.auth_v2</code></td><td><code>true</code> <br/> <code>false</code></td><td>The value of a particular feature flag for this request</td></tr><tr><td><code>feature_flag.double_write_to_new_db</code></td><td><code>true</code> <br/> <code>false</code></td><td>The value of a particular feature flag for this request</td></tr></tbody></table></div>
|
||
<blockquote>
|
||
<p>What errors are the users in the new authentication flows hitting? How does it compare to the control group?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> COUNT</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span><span style="color:#F97583"> AND</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> feature_flag</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">auth_v2</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">exception</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">slug</span></span></code></pre>
|
||
<h3 id="versions-of-important-things">Versions of important things<a class="heading-anchor" aria-label="Link to this section" href="#versions-of-important-things">#</a></h3>
|
||
<p>Runtimes, frameworks, and any major libraries you are using can be really helpful context.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>go.version</code></td><td><code>go1.23.2</code></td><td>What version of your language runtime are you using?</td></tr><tr><td><code>rails.version</code></td><td><code>7.2.1.1</code></td><td>Pick out any core libraries like web frameworks and track their version too</td></tr><tr><td><code>postgres.version</code></td><td><code>16.4</code></td><td>If you can add the versions of any datastores you’re using, even better</td></tr></tbody></table></div>
|
||
<blockquote>
|
||
<p>A security issue with Rails just got announced. What versions of the framework are our services using?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> COUNT_DISTINCT(</span><span style="color:#79B8FF">service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">environment</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "production"</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> rails</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">version</span></span></code></pre>
|
||
<blockquote>
|
||
<p>Our memory usage seems higher that it used to be. Didn’t we upgrade the runtime recently? Does that correlate?</p>
|
||
</blockquote>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="sql"><code><span class="line"><span style="color:#F97583">SELECT</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> HEATMAP(</span><span style="color:#79B8FF">metrics</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">memory_mb</span><span style="color:#E1E4E8">)</span></span>
|
||
<span class="line"><span style="color:#F97583">WHERE</span></span>
|
||
<span class="line"><span style="color:#E1E4E8"> main </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> true </span><span style="color:#F97583">AND</span></span>
|
||
<span class="line"><span style="color:#79B8FF"> service</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">name</span><span style="color:#F97583"> =</span><span style="color:#9ECBFF"> "api-service"</span></span>
|
||
<span class="line"><span style="color:#F97583">GROUP BY</span><span style="color:#79B8FF"> go</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">version</span></span></code></pre>
|
||
<h3 id="your-specific-application">Your specific application<a class="heading-anchor" aria-label="Link to this section" href="#your-specific-application">#</a></h3>
|
||
<p>Now we go off the map and get to the really valuable stuff. Your app likely does something unique or works in a particular
|
||
domain. You might need to <em>really</em> care about which professional credentials a Dentist using your app has, or which
|
||
particular storage warehouse a package is in, or which chip is in the embedded tracking device installed in the cat that
|
||
your app exists to track.</p>
|
||
<p>No framework is going to be able to understand what parts of your domain are important to track and automate this for you,
|
||
you have to do that.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="table-scroll" tabindex="0"><table><thead><tr><th>Attribute</th><th>Examples</th><th>Description</th></tr></thead><tbody><tr><td><code>asset_upload.s3_bucket_path</code></td><td><code>s3://bucket-name/path/to/asset.jpg</code></td><td>If you upload something, add context about where</td></tr><tr><td><code>email_vendor.transaction_id</code></td><td><code>62449c60-b51e-4d5c-8464-49217d91c441</code></td><td>If you interact with a vendor, track whatever transaction ID they give you in case you need to follow up with them</td></tr><tr><td><code>vcs_integration.vendor</code></td><td><code>github</code> <br/> <code>gitlab</code> <br/> <code>bitbucket</code></td><td>If there are 3-4 types that something might fall into, be sure to add that context. Ex: If 2% of requests start failing because bitbucket is experiencing issues, this will help identify the source of the issue immediately.</td></tr><tr><td><code>process_submission.queue_length</code></td><td><code>153</code> <br/> <code>1</code></td><td>Any time you interact with a queue, see if you can get the current length during submission</td></tr></tbody></table></div>
|
||
<h2 id="things-to-note">Things to note<a class="heading-anchor" aria-label="Link to this section" href="#things-to-note">#</a></h2>
|
||
<h3 id="you-should-probably-add-the-thing">You should probably add the thing<a class="heading-anchor" aria-label="Link to this section" href="#you-should-probably-add-the-thing">#</a></h3>
|
||
<p>If you find yourself asking “Am I ever really going to need this bit of data?”, default to throwing the attribute on.
|
||
The marginal cost of each extra attribute is very small. If the data volume does start to grow, prefer wider,
|
||
more context-rich events and a higher sample rate vs smaller events with a lower sample rate.</p>
|
||
<h3 id="heatmaps-are-your-friend">Heatmaps are your friend<a class="heading-anchor" aria-label="Link to this section" href="#heatmaps-are-your-friend">#</a></h3>
|
||
<p><a href="https://www.honeycomb.io/blog/heatmaps-are-the-new-hotness">Honeycomb’s heatmaps</a> are amazing at helping you find outliers, seeing multi-modal distributions,
|
||
and getting a feel for your data. I wish more tooling supported them. I am not sure I can build software without them any more.</p>
|
||
<h3 id="embrace-the-feedback-loop">Embrace the feedback loop<a class="heading-anchor" aria-label="Link to this section" href="#embrace-the-feedback-loop">#</a></h3>
|
||
<p>When you are modifying code, make a change to the telemetry so that you can see the impact of the new code running. Once
|
||
the code is released, check to make sure that you see the outcome you expected. Don’t hesitate to add specific fields
|
||
for one release and them remove them after.</p>
|
||
<p>Tighter feedback loops are like going faster on a bicycle. They make for more stable systems and let you move faster
|
||
with confidence.</p>
|
||
<h3 id="semantic-conventions-and-naming-consistency">Semantic conventions and naming consistency<a class="heading-anchor" aria-label="Link to this section" href="#semantic-conventions-and-naming-consistency">#</a></h3>
|
||
<p>I’ve tried to embrace <a href="https://opentelemetry.io/docs/specs/semconv/general/trace/">semantic conventions</a> in my naming, but would not
|
||
be surprised if I’ve made multiple errors. Naming is hard!</p>
|
||
<p>It’s also hard to get consistency right within an organization or even across multiple systems owned by the same team. I would recommend
|
||
trying to use semantic conventions as a guide, but do prioritize getting data out of your system in some form and getting some early wins
|
||
over exacting adherence to an evolving specification. Once this data has proven its value within your organization, then you will have the
|
||
leverage to spend engineering cycles on making things consistent.</p>
|
||
<p>In the long run semantic conventions should allow Observability vendors to build new value and understanding on top of the telemetry you emit,
|
||
but this effort is only just getting started.</p>
|
||
<h2 id="frequent-objections">Frequent Objections<a class="heading-anchor" aria-label="Link to this section" href="#frequent-objections">#</a></h2>
|
||
<h3 id="does-this-really-work">Does this really work??<a class="heading-anchor" aria-label="Link to this section" href="#does-this-really-work">#</a></h3>
|
||
<p>I have done this for dozens of production systems. Every single time the data has been invaluable for digging in and understanding what the
|
||
system is <em>actually</em> doing, and we’ve found something surprising, even for the engineers who had worked on the system for many years.</p>
|
||
<p>Things like:</p>
|
||
<ul>
|
||
<li>Oh, actually 90% of the traffic of this system comes from one user</li>
|
||
<li>Wait, one of our worker processes is actually running a month old version of the code somehow?</li>
|
||
<li>This API endpoint usually has payloads of 1-2kb, but there is an edge case affecting one user where it’s 40+MB. This causes their page loads to be <strong>several minutes longer than the p99</strong>.</li>
|
||
<li>After instrumenting the authentication middleware, around 20% of requests still didn’t have user info. There was a whole second authentication system for a different class of users that hadn’t been touched in years.</li>
|
||
<li>This endpoint that we’d like to deprecate accepts data in the form of <code>A</code>, <code>B</code>, and <code>C</code>, but none of our traffic ever even uses <code>C</code>. We can just drop support for that now.</li>
|
||
</ul>
|
||
<h3 id="i-dont-like-it-this-feels-wrong">I don’t like it. This feels wrong<a class="heading-anchor" aria-label="Link to this section" href="#i-dont-like-it-this-feels-wrong">#</a></h3>
|
||
<p>For anyone feeling that way now, I ask you to <a href="https://signalvnoise.com/posts/3124-give-it-five-minutes">give it five minutes</a>.</p>
|
||
<p>I find that when a log line wraps around your terminal window multiple times, most developers have a pretty visceral negative reaction.</p>
|
||
<p>This <em>feels</em> right:</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="log"><code><span class="line"><span style="color:#E1E4E8">[</span><span style="color:#6A737D">2024-09-18</span><span style="color:#6A737D"> 22:48:32.990</span><span style="color:#E1E4E8">] Request started http_path=/v1/charges request_id=req_123</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">[</span><span style="color:#6A737D">2024-09-18</span><span style="color:#6A737D"> 22:48:32.991</span><span style="color:#E1E4E8">] User authenticated auth_type=api_key key_id=mk_123 user_id=usr_123</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">[</span><span style="color:#6A737D">2024-09-18</span><span style="color:#6A737D"> 22:48:32.992</span><span style="color:#E1E4E8">] Rate limiting ran rate_allowed=</span><span style="color:#79B8FF">true</span><span style="color:#E1E4E8"> rate_quota=</span><span style="color:#79B8FF">100</span><span style="color:#E1E4E8"> rate_remaining=</span><span style="color:#79B8FF">99</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">[</span><span style="color:#6A737D">2024-09-18</span><span style="color:#6A737D"> 22:48:32.998</span><span style="color:#E1E4E8">] Charge created charge_id=ch_123 permissions_used=account_write request_id=req_123</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">[</span><span style="color:#6A737D">2024-09-18</span><span style="color:#6A737D"> 22:48:32.999</span><span style="color:#E1E4E8">] Request finished http_status=</span><span style="color:#79B8FF">200</span><span style="color:#E1E4E8"> request_id=req_123</span></span></code></pre>
|
||
<p>But this <em>feels</em> wrong:</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="log"><code><span class="line"><span style="color:#E1E4E8">[</span><span style="color:#6A737D">2024-10-20T14:43:36.851Z</span><span style="color:#E1E4E8">] duration_ms=</span><span style="color:#79B8FF">1266</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">1819686777117</span><span style="color:#E1E4E8"> main=</span><span style="color:#79B8FF">true</span><span style="color:#79B8FF"> http.ip_address</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">92</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">21</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">101</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">252</span><span style="color:#79B8FF"> instance.id</span><span style="color:#E1E4E8">=api-</span><span style="color:#79B8FF">1</span><span style="color:#79B8FF"> instance.memory_mb</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">12336</span></span>
|
||
<span class="line"><span style="color:#79B8FF">instance.cpu_count</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">4</span><span style="color:#79B8FF"> instance.type</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">t3.small</span><span style="color:#79B8FF"> http.request.method</span><span style="color:#E1E4E8">=GET </span><span style="color:#79B8FF">http.request.path</span><span style="color:#E1E4E8">=/api/categories/substantia-trado</span></span>
|
||
<span class="line"><span style="color:#79B8FF">http.route</span><span style="color:#E1E4E8">=/api/categories/:slug </span><span style="color:#79B8FF">http.request.body.size</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">293364</span><span style="color:#79B8FF"> http.request.header.content_type</span><span style="color:#E1E4E8">=application/xml</span></span>
|
||
<span class="line"><span style="color:#79B8FF">user_agent.original</span><span style="color:#E1E4E8">=</span><span style="color:#9ECBFF">"Mozilla/5.0 (X11; Linux i686 AppleWebKit/535.1.2 (KHTML, like Gecko) Chrome/39.0.826.0 Safari/535.1.2"</span><span style="color:#79B8FF"> user_agent.device</span><span style="color:#E1E4E8">=phone</span></span>
|
||
<span class="line"><span style="color:#79B8FF">user_agent.os</span><span style="color:#E1E4E8">=Windows </span><span style="color:#79B8FF">user_agent.browser</span><span style="color:#E1E4E8">=Edge </span><span style="color:#79B8FF">user_agent.browser_version</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">3</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">0</span><span style="color:#79B8FF"> url.scheme</span><span style="color:#E1E4E8">=https </span><span style="color:#79B8FF">url.host</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">api-service.com</span><span style="color:#79B8FF"> service.name</span><span style="color:#E1E4E8">=api-service</span></span>
|
||
<span class="line"><span style="color:#79B8FF">service.version</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">1</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">0</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">0</span><span style="color:#79B8FF"> build.id</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">1234567890</span><span style="color:#79B8FF"> go.version</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">go1.23.2</span><span style="color:#79B8FF"> rails.version</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">7</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">2</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">1</span><span style="color:#E1E4E8">.</span><span style="color:#79B8FF">1</span><span style="color:#79B8FF"> service.environment</span><span style="color:#E1E4E8">=production </span><span style="color:#79B8FF">service.team</span><span style="color:#E1E4E8">=api-team</span></span>
|
||
<span class="line"><span style="color:#79B8FF">service.slack_channel</span><span style="color:#E1E4E8">=#api-alerts </span><span style="color:#79B8FF">service.build.deployment.at</span><span style="color:#E1E4E8">=</span><span style="color:#6A737D">2024-10-14T19:47:38Z</span></span>
|
||
<span class="line"><span style="color:#79B8FF">service.build.diff_url</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">https://github.com/your-company/api-service/compare/c9d9380..05e5736</span></span>
|
||
<span class="line"><span style="color:#79B8FF">service.build.pull_request_url</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">https://github.com/your-company/api-service/pull/123</span></span>
|
||
<span class="line"><span style="color:#79B8FF">service.build.git_hash</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">05e5736</span><span style="color:#79B8FF"> service.build.deployment.user</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">keanu.reeves</span><span style="color:#E1E4E8">@</span><span style="color:#79B8FF">your-company.com</span></span>
|
||
<span class="line"><span style="color:#79B8FF">service.build.deployment.trigger</span><span style="color:#E1E4E8">=manual </span><span style="color:#79B8FF">container.id</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">1234567890</span><span style="color:#79B8FF"> container.name</span><span style="color:#E1E4E8">=api-service-</span><span style="color:#79B8FF">1234567890</span><span style="color:#79B8FF"> cloud.availability_zone</span><span style="color:#E1E4E8">=us-east-</span><span style="color:#79B8FF">1</span></span>
|
||
<span class="line"><span style="color:#79B8FF">cloud.region</span><span style="color:#E1E4E8">=us-east-</span><span style="color:#79B8FF">1</span><span style="color:#79B8FF"> k8s.pod.name</span><span style="color:#E1E4E8">=api-service-</span><span style="color:#79B8FF">1234567890</span><span style="color:#79B8FF"> k8s.cluster.name</span><span style="color:#E1E4E8">=api-service-cluster </span><span style="color:#79B8FF">feature_flag.auth_v2</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">true</span></span>
|
||
<span class="line"><span style="color:#79B8FF">http.response.status_code</span><span style="color:#E1E4E8">=</span><span style="color:#79B8FF">401</span><span style="color:#79B8FF"> user.id</span><span style="color:#E1E4E8">=Samanta27@</span><span style="color:#79B8FF">gmail.com</span><span style="color:#79B8FF"> user.type</span><span style="color:#E1E4E8">=vip </span><span style="color:#79B8FF">user.auth_method</span><span style="color:#E1E4E8">=sso-google </span><span style="color:#79B8FF">user.team_id</span><span style="color:#E1E4E8">=team-</span><span style="color:#79B8FF">1</span></span></code></pre>
|
||
<p><strong>You are structuring data so that it can be read efficiently by machines, not humans.</strong> Our systems emit too much data to waste
|
||
precious human lifetimes using our eyeballs to scan lines of text looking for patterns to jump out. Let the robots help.</p>
|
||
<h3 id="this-seems-like-a-lot-of-work">This seems like a lot of work<a class="heading-anchor" aria-label="Link to this section" href="#this-seems-like-a-lot-of-work">#</a></h3>
|
||
<p>If you want to implement everything I’ve talked about in this post that would be a <em>ton</em> of work. However, even implementing
|
||
the easiest subset is going to provide a lot of value. Not doing this results in so much <em>more work</em> building a mental model
|
||
of your system, trying to debug by thinking through the code and hoping your mental model matches reality.</p>
|
||
<p>A lot of this logic can be put into shared libraries within your org, though getting them adopted, keeping them updated and
|
||
in-sync, and getting engineers used to these tools presents a whole different set of challenges.</p>
|
||
<p>Many of these things could be surfaced to you by opinionated platforms or frameworks. I would love to see things move in
|
||
this direction.</p>
|
||
<h3 id="isnt-this-a-lot-of-data-wont-it-cost-a-lot">Isn’t this a lot of data? Won’t it cost a lot??<a class="heading-anchor" aria-label="Link to this section" href="#isnt-this-a-lot-of-data-wont-it-cost-a-lot">#</a></h3>
|
||
<p><a href="https://news.ycombinator.com/item?id=39531022"><img src="/_astro/hn-comment-on-cost.BvDY3dCa_Z1n5NEx.webp" alt="Hacker News comment: This isn't an unknown idea outside of Meta, it's just really expensive, especially if you're using a vendor and not building your own tooling. Prohibitively so, even with sampling." loading="lazy" decoding="async" width="1472" height="262"></a></p>
|
||
<p>First, you should compare this to your current log volume per request. I have seen many systems where this
|
||
approach would <em>reduce</em> overall log volume.</p>
|
||
<p>However storing this data for every request against your system could be too expensive at scale. That’s
|
||
where sampling comes in. Sampling gives you the controls to determine what you want to spend vs the value
|
||
you receive from storing and making that data available to query.</p>
|
||
<p>Realtime OLAP systems are also getting cheaper all the time. Once upon a time Scuba held all data in memory
|
||
to make these types of questions quick to answer. Now most OLAP systems are evolving to columnar files stored
|
||
on cloud object storage with queries handled by ephemeral compute which is many orders of magnitude cheaper.</p>
|
||
<p>In the next section I’ll show just how much cheaper.</p>
|
||
<h3 id="repeated-data">Repeated data<a class="heading-anchor" aria-label="Link to this section" href="#repeated-data">#</a></h3>
|
||
<blockquote>
|
||
<p>Many of these fields will be the same for every request. Isn’t that really inefficient?</p>
|
||
</blockquote>
|
||
<p>This is where our intuitions can lie to us. Let’s look at a concrete example.</p>
|
||
<p>I <a href="https://github.com/jmorrell/a-practitioners-guide-to-wide-events/blob/main/column-storage-compression/index.js">wrote a script</a><sup><a href="#user-content-fn-who-wrote-it" id="user-content-fnref-who-wrote-it" data-footnote-ref aria-describedby="footnote-label">1</a></sup>
|
||
to generate a newline-delimited JSON file with a lot of the above fields and at least somewhat reasonable fake values.</p>
|
||
<p>Let’s say our service is serving <code>1000</code> req/s all day and sampling 1% of that traffic. Rounding to a whole number, that’s about a
|
||
million events. Generating a million example wide events results in a <code>1.6GB</code> file.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="js"><code><span class="line"><span style="color:#E1E4E8">http_logs.ndjson </span><span style="color:#79B8FF">1607.61</span><span style="color:#79B8FF"> MB</span></span></code></pre>
|
||
<p>But we repeat the keys on every single line. Even just turning it into a CSV cuts the size by
|
||
more than 50%.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="js"><code><span class="line"><span style="color:#E1E4E8">http_logs.csv </span><span style="color:#79B8FF">674.72</span><span style="color:#79B8FF"> MB</span></span></code></pre>
|
||
<p>Gzipping the file shows an amazing amount of compression, hinting that this isn’t
|
||
as much data as we might think.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="js"><code><span class="line"><span style="color:#E1E4E8">http_logs.ndjson.gz </span><span style="color:#79B8FF">101.67</span><span style="color:#79B8FF"> MB</span></span></code></pre>
|
||
<p>Column store formats like <code>parquet</code> and Duckdb’s native format can do even better.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="js"><code><span class="line"><span style="color:#E1E4E8">http_logs.parquet </span><span style="color:#79B8FF">88.83</span><span style="color:#79B8FF"> MB</span></span>
|
||
<span class="line"><span style="color:#E1E4E8">http_logs.duckdb </span><span style="color:#79B8FF">80.01</span><span style="color:#79B8FF"> MB</span></span></code></pre>
|
||
<p>They store all of the data for a specific column contiguously, which lends itself to different
|
||
compression approachs. In the simplest case, if the column is always the same value, it can store that fact only once.
|
||
Values that are the same across an entire <a href="https://cloudsqale.com/2020/05/29/how-parquet-files-are-written-row-groups-pages-required-memory-and-flush-operations/">row group</a>
|
||
are incredibly cheap.</p>
|
||
<p><img src="/_astro/constant.wLvXM20h_Z1gberq.webp" alt="DuckDB diagram showing how a constant value along a whole column gets compressed" loading="lazy" decoding="async" width="400" height="225"></p>
|
||
<p>If there are 2-3 different values, it can use dictionary-encoding to bit-pack these values really tightly. This also
|
||
speeds up queries against this column.</p>
|
||
<p><img src="/_astro/dictionary.B0WbkNfy_Z1ABiFj.webp" alt="DuckDB diagram showing how values get compressed using dictionary encoding" loading="lazy" decoding="async" width="400" height="225"></p>
|
||
<p><a href="https://duckdb.org/2022/10/28/lightweight-compression.html">DuckDB has a great writeup on this</a> which goes into much more detail.
|
||
All of the data remains available and is easily (and quickly!) queryable.</p>
|
||
<p>This is hardly “big data”. Storing this on <a href="https://developers.cloudflare.com/r2/pricing/">Cloudflare’s R2</a> for a month would cost <code>US$ 0.0012</code>.
|
||
You could keep 60 days of retention for <code>US$ 0.072</code> / month.</p>
|
||
<pre class="astro-code github-dark" style="background-color:#24292e;color:#e1e4e8;overflow-x:auto" tabindex="0" data-language="plaintext"><code><span class="line"><span>❯ duckdb http_logs.duckdb</span></span>
|
||
<span class="line"><span>D SELECT COUNT(*) FROM http_logs;</span></span>
|
||
<span class="line"><span>┌──────────────┐</span></span>
|
||
<span class="line"><span>│ count_star() │</span></span>
|
||
<span class="line"><span>│ int64 │</span></span>
|
||
<span class="line"><span>├──────────────┤</span></span>
|
||
<span class="line"><span>│ 1000000 │</span></span>
|
||
<span class="line"><span>└──────────────┘</span></span>
|
||
<span class="line"><span>Run Time (s): real 0.002 user 0.002350 sys 0.000946</span></span>
|
||
<span class="line"><span>D SELECT SUM(duration_ms) FROM http_logs;</span></span>
|
||
<span class="line"><span>┌───────────────────┐</span></span>
|
||
<span class="line"><span>│ sum(duration_ms) │</span></span>
|
||
<span class="line"><span>│ double │</span></span>
|
||
<span class="line"><span>├───────────────────┤</span></span>
|
||
<span class="line"><span>│ 999938387.7714149 │</span></span>
|
||
<span class="line"><span>└───────────────────┘</span></span>
|
||
<span class="line"><span>Run Time (s): real 0.003 user 0.008020 sys 0.000415</span></span></code></pre>
|
||
<p>There are even <a href="https://arrow.apache.org/">in-memory</a> and <a href="https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/">transport formats</a> to help
|
||
reduce size in memory and on the wire. <a href="https://opentelemetry.io/blog/2023/otel-arrow">OpenTelemetry is adopting arrow for its payloads</a> for this reason.</p>
|
||
<p>I found <a href="https://www.youtube.com/watch?v=dlO1cKnfWAI">this podcast on the FDAP stack particularly helpful in understanding this space</a>.</p>
|
||
<h3 id="couldnt-we-join-data-from-multiple-spans-together-to-get-this-information-query-the-whole-trace-at-once">Couldn’t we <code>JOIN</code> data from multiple spans together to get this information? Query the whole trace at once?<a class="heading-anchor" aria-label="Link to this section" href="#couldnt-we-join-data-from-multiple-spans-together-to-get-this-information-query-the-whole-trace-at-once">#</a></h3>
|
||
<p>This is certainly possible. <a href="https://docs.honeycomb.io/investigate/query/build/#clauses">Honeycomb has started allowing you to filter on fields on other spans in the same trace</a>.
|
||
However I’d qualify this as very advanced. You want to make the right thing the easiest thing, and if you make it
|
||
harder to ask questions, people will simply ask fewer questions. There are already a million things competing for our
|
||
attention. Keep it simple. Make it fast.</p>
|
||
<h3 id="does-this-mean-i-dont-need-metrics">Does this mean I don’t need metrics?<a class="heading-anchor" aria-label="Link to this section" href="#does-this-mean-i-dont-need-metrics">#</a></h3>
|
||
<p>You should probably still generate high-level metrics, though I bet you will need far fewer.</p>
|
||
<p>Metrics are great when you know you want an exact answer to a very specific question that you know ahead of time.
|
||
Questions like “How many requests did a serve yesterday?” or “What was my CPU usage like last month?”</p>
|
||
<section data-footnotes class="footnotes"><h2 class="sr-only" id="footnote-label">Footnotes<a class="heading-anchor" aria-label="Link to this section" href="#footnote-label">#</a></h2>
|
||
<ol>
|
||
<li id="user-content-fn-who-wrote-it"><a class="footnote-number" href="#user-content-fnref-who-wrote-it" aria-label="Back to reference 1">1.</a>
|
||
<p>Well… mostly <a href="https://www.cursor.com/">Cursor</a> wrote it <a href="#user-content-fnref-who-wrote-it" data-footnote-backref aria-label="Back to reference 1" class="data-footnote-backref">↩</a></p>
|
||
</li>
|
||
</ol>
|
||
</section></article><div class="measure mt-14"><section id="newsletter-signup-bottom" class="not-prose grid gap-5 border-y border-black/10 py-8 sm:grid-cols-[minmax(0,1fr)_minmax(18rem,1fr)] sm:items-center sm:gap-10 dark:border-white/10" aria-labelledby="newsletter-signup-bottom-heading"><div><h2 id="newsletter-signup-bottom-heading" class="text-lg font-semibold tracking-tight text-black dark:text-white">New writing, occasionally.</h2><p class="mt-1 max-w-md text-sm leading-relaxed text-black/60 dark:text-white/55">Get my posts in your inbox. No fixed schedule, no noise.</p></div><form action="https://buttondown.com/api/emails/embed-subscribe/jeremymorrell" method="post" class="newsletter-form"><label for="newsletter-signup-bottom-email" class="sr-only">Email address</label><div class="flex flex-col gap-2 min-[28rem]:flex-row"><input id="newsletter-signup-bottom-email" name="email" type="email" autocomplete="email" inputmode="email" placeholder="you@example.com" required class="min-w-0 flex-1 rounded-md border border-black/15 bg-white/55 px-3.5 py-2.5 text-base text-black shadow-sm outline-none placeholder:text-black/35 focus:border-black/40 focus:ring-2 focus:ring-black/10 dark:border-white/15 dark:bg-white/[0.06] dark:text-white dark:placeholder:text-white/30 dark:focus:border-white/40 dark:focus:ring-white/10"><button type="submit" class="cursor-pointer rounded-md bg-stone-900 px-4 py-2.5 text-sm font-semibold whitespace-nowrap text-white transition-colors hover:bg-stone-700 focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-stone-900 dark:bg-stone-100 dark:text-stone-900 dark:hover:bg-white dark:focus-visible:outline-stone-100">Subscribe</button></div></form></section></div></div></main><footer class="animate"><div class="mx-auto max-w-screen-md px-5"><div class="flex justify-between items-center"><div>© 2026 | Jeremy Morrell</div></div></div></footer><script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"e70ec7b1b71b41478b1ee73c8ea74a1b","r":1,"spa":2}' crossorigin="anonymous"></script>
|
||
</body></html> |