Files
nexus/sreweekly/articles/523/07-instrumenting-at-10-years-per-second.html
2026-09-12 17:23:01 +08:00

114 lines
148 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="image" href="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg"/><link rel="stylesheet" href="/_next/static/css/bc5c0eaecd39bc7d.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/7ffcc02a3c67389b.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/c20e76e7d11dec32.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/a6eb8c1f37be68d5.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-d716a75e4017380e.js"/><script src="/_next/static/chunks/4bd1b696-100b9d70ed4e49c1.js" async=""></script><script src="/_next/static/chunks/9da6db1e-8a4e8c23641c31bf.js" async=""></script><script src="/_next/static/chunks/1255-a242e0ce0372fddf.js" async=""></script><script src="/_next/static/chunks/main-app-bda8a52fe841b3aa.js" async=""></script><script src="/_next/static/chunks/4554-984b480a9db5f3fe.js" async=""></script><script src="/_next/static/chunks/7563-4dfa0994a9b0e37c.js" async=""></script><script src="/_next/static/chunks/9216-9272709ba40c1cfd.js" async=""></script><script src="/_next/static/chunks/3874-323555cd5c106ba4.js" async=""></script><script src="/_next/static/chunks/3091-bf1537ff6571a1bd.js" async=""></script><script src="/_next/static/chunks/app/layout-f61a55efa656d91f.js" async=""></script><script src="/_next/static/chunks/app/not-found-b861935655c5698b.js" async=""></script><script src="/_next/static/chunks/5001-fae0bebdd93f95e3.js" async=""></script><script src="/_next/static/chunks/2423-c8b1388240338d7d.js" async=""></script><script src="/_next/static/chunks/652-f40dcb480b4e356f.js" async=""></script><script src="/_next/static/chunks/730-8191af9e9424b148.js" async=""></script><script src="/_next/static/chunks/app/page-cc18ae1a3a7a5ed8.js" async=""></script><script src="/_next/static/chunks/0dbeb660-9e82e99813552576.js" async=""></script><script src="/_next/static/chunks/1545-7abf96433ff6553f.js" async=""></script><script src="/_next/static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js" async=""></script><meta name="next-size-adjust" content=""/><title>Accelerating the pace of custom instrumentation</title><meta name="description" content="Struggling to instrument dozens of microservices? Discover our config-driven approach that accelerates custom instrumentation from years to weeks."/><link rel="canonical" href="https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second"/><meta property="og:title" content="Accelerating the pace of custom instrumentation"/><meta property="og:description" content="Struggling to instrument dozens of microservices? Discover our config-driven approach that accelerates custom instrumentation from years to weeks."/><meta property="og:url" content="https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second"/><meta property="og:site_name" content="Perspective Unspoken"/><meta property="og:image" content="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg"/><meta property="og:image:width" content="1200"/><meta property="og:image:height" content="630"/><meta property="og:type" content="article"/><meta property="article:published_time" content="2025-12-23T04:10:27.099Z"/><meta property="article:author" content="https://jaywhy13.hashnode.dev/@jaywhy13"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:creator" content="https://twitter.com/kramnaej"/><meta name="twitter:title" content="Accelerating the pace of custom instrumentation"/><meta name="twitter:description" content="Struggling to instrument dozens of microservices? Discover our config-driven approach that accelerates custom instrumentation from years to weeks."/><meta name="twitter:image" content="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg"/><link rel="icon" href="/favicon.ico"/><script type="text/javascript">window.addEventListener('message',function(e){var d=e.data;if(!d||d.__hnWidget!==1||typeof d.h!=='number'||d.h<=0)return;var f=document.querySelectorAll('.hn-widget-frame');for(var i=0;i<f.length;i++){if(f[i].contentWindow===e.source){f[i].style.height=d.h+'px';break;}}});</script><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body class="__variable_ede2c9 __variable_558dcf bg-background dark:bg-background"><div hidden=""><!--$--><!--/$--></div><script>((a2, b2, c4, d3, e2, f2, g4, h3) => {
let i3 = document.documentElement, j2 = ["light", "dark"];
function k2(b3) {
var c5;
(Array.isArray(a2) ? a2 : [a2]).forEach((a3) => {
let c6 = a3 === "class", d4 = c6 && f2 ? e2.map((a4) => f2[a4] || a4) : e2;
c6 ? (i3.classList.remove(...d4), i3.classList.add(f2 && f2[b3] ? f2[b3] : b3)) : i3.setAttribute(a3, b3);
}), c5 = b3, h3 && j2.includes(c5) && (i3.style.colorScheme = c5);
}
if (d3) k2(d3);
else try {
let a3 = localStorage.getItem(b2) || c4, d4 = g4 && a3 === "system" ? window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light" : a3;
k2(d4);
} catch {
}
})("class","theme","system",null,["light","dark"],null,true,true)</script><main id="main-content" class="min-h-screen bg-transparent"><header class="sticky top-0 z-50 w-full py-4 bg-background backdrop-blur-sm transition-transform duration-300 translate-y-0"><a href="#main-content" class="sr-only focus:not-sr-only focus:absolute focus:left-4 focus:top-4 focus:z-50 focus:rounded-md focus:bg-background focus:px-4 focus:py-2 focus:text-sm focus:shadow">Skip to main content</a><div class="relative w-full mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem] px-4 sm:px-12 md:px-18 lg:px-16 xl:px-10"><div class="flex items-center justify-between gap-3 xl:hidden"><div class="flex items-center gap-3 min-w-0 flex-1"><a href="https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev&amp;utm_medium=referral&amp;utm_campaign=blog_header_logo&amp;utm_content=logo" class="inline-flex items-center justify-center text-lg text-muted-foreground hover:text-foreground transition border rounded-md h-9 w-9 shrink-0 border-border hover:bg-muted"><i class="text-xl fa-brands fa-hashnode" aria-hidden="true"></i><span class="sr-only">Hashnode</span></a><a title="Perspective Unspoken" class="text-lg font-semibold text-foreground truncate min-w-0" href="/">Perspective Unspoken</a></div><div class="flex items-center gap-2 shrink-0"><button class="gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent py-2 h-9 w-9 px-0 flex items-center justify-center text-muted-foreground hover:text-foreground bg-transparent border-border shadow-none" aria-label="Open search (press Control or Command and K)"><i class="fa-jelly fa-search text-sm" aria-hidden="true"></i><span class="sr-only">Open search (press Control or Command and K)</span></button><button class="inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent w-9 px-3 h-9 text-muted-foreground hover:text-foreground shadow-none" type="button" id="radix-_R_2j4anpfalb_" aria-haspopup="menu" aria-expanded="false" data-state="closed" data-slot="dropdown-menu-trigger"><i class="fa-jelly fa-sun text-sm scale-100 rotate-0 transition-all dark:scale-0 dark:-rotate-90" aria-hidden="true"></i><i class="fa-jelly fa-moon absolute text-sm scale-0 rotate-90 transition-all dark:scale-100 dark:rotate-0" aria-hidden="true"></i><span class="sr-only">Toggle theme</span></button><button class="inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 border border-input bg-background hover:bg-accent hover:text-accent-foreground py-2 h-9 w-10 px-0 shadow-none" type="button" aria-haspopup="dialog" aria-expanded="false" aria-controls="radix-_R_3j4anpfalb_" data-state="closed" data-slot="sheet-trigger"><i class="fa-jelly fa-bars" aria-hidden="true"></i><span class="sr-only">Open menu</span></button></div></div><div class="hidden xl:grid gap-4 items-center xl:grid-cols-[1fr_auto_1fr] xl:gap-6 max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="flex items-center w-full gap-3 min-w-0"><a href="https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev&amp;utm_medium=referral&amp;utm_campaign=blog_header_logo&amp;utm_content=logo" class="inline-flex items-center justify-center text-lg text-muted-foreground hover:text-foreground transition border rounded-md h-9 w-9 border-border hover:bg-muted"><i class="text-xl fa-brands fa-hashnode" aria-hidden="true"></i><span class="sr-only">Hashnode</span></a><span class="h-6 border-l border-border"></span><p class="font-semibold text-foreground min-w-0 truncate text-lg sm:text-xl"><a title="Perspective Unspoken" href="/">Perspective Unspoken</a></p></div><nav aria-label="Main navigation" class="flex justify-center w-full"><nav aria-label="Main" data-orientation="horizontal" dir="ltr" data-slot="navigation-menu" data-viewport="false" class="group/navigation-menu relative flex flex-1 items-center max-w-max justify-center"><div style="position:relative"><ul data-orientation="horizontal" data-slot="navigation-menu-list" class="group flex flex-1 list-none items-center flex-nowrap justify-center gap-1 text-muted-foreground text-base" dir="ltr"><li data-slot="navigation-menu-item" class="relative"><a target="_blank" rel="noopener" data-slot="navigation-menu-link" class="data-[active=true]:focus:bg-accent data-[active=true]:hover:bg-accent data-[active=true]:bg-accent/50 data-[active=true]:text-accent-foreground focus:bg-accent focus:text-accent-foreground [&amp;_svg:not([class*=&#x27;text-&#x27;])]:text-muted-foreground flex-col p-2 outline-none focus-visible:outline-1 [&amp;_svg:not([class*=&#x27;size-&#x27;])]:size-4 inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent hover:text-accent-foreground h-9 px-4 py-2" data-radix-collection-item="" href="https://calendar.google.com/calendar/u/0/appointments/AcZssZ3tWQd0YSITufTZaAitKeE6Ln2NBOouN5Qlop4=?gv=true">Calendar</a></li></ul></div></nav></nav><div class="flex items-center justify-end gap-3 text-muted-foreground"><button class="gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent py-2 h-9 w-9 px-0 flex items-center justify-center text-muted-foreground hover:text-foreground bg-transparent border-border shadow-none" aria-label="Open search (press Control or Command and K)"><i class="fa-jelly fa-search text-sm" aria-hidden="true"></i><span class="sr-only">Open search (press Control or Command and K)</span></button><div class="flex items-center gap-3"><button class="inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent w-9 px-3 h-9 text-muted-foreground hover:text-foreground shadow-none" type="button" id="radix-_R_6t4anpfalb_" aria-haspopup="menu" aria-expanded="false" data-state="closed" data-slot="dropdown-menu-trigger"><i class="fa-jelly fa-sun text-sm scale-100 rotate-0 transition-all dark:scale-0 dark:-rotate-90" aria-hidden="true"></i><i class="fa-jelly fa-moon absolute text-sm scale-0 rotate-90 transition-all dark:scale-100 dark:rotate-0" aria-hidden="true"></i><span class="sr-only">Toggle theme</span></button><a target="_blank" rel="noopener" class="inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 border border-input bg-background hover:bg-accent hover:text-accent-foreground py-2 gap-2 px-3 h-9 shadow-none" href="https://hashnode.com"><i class="text-sm fa-jelly fa-pencil" aria-hidden="true"></i>Write</a></div></div></div></div></header><div data-slot="dialog-header" class="flex flex-col gap-2 text-center sm:text-left sr-only"><h2 id="radix-_R_8anpfalbH1_" data-slot="dialog-title" class="text-lg leading-none font-semibold">Command Palette</h2><p id="radix-_R_8anpfalbH2_" data-slot="dialog-description" class="text-muted-foreground text-sm">Search for a command to run...</p></div><section class="py-8"><div class="px-4 sm:px-8"><div class="w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="grid gap-5 max-w-full"><article class="w-full max-w-full rounded-xl border bg-card text-card-foreground"><div class="px-5 pt-12 pb-8 sm:px-8"><header class="space-y-5"><div class="space-y-4 text-center max-w-4xl mx-auto"><h1 class="text-2xl sm:text-3xl md:text-4xl lg:text-5xl font-bold leading-tight tracking-tight text-balance text-foreground">Instrumenting at 10 years per second</h1><p class="text-lg sm:text-xl leading-normal font-normal text-muted-foreground text-balance">How we made custom instrumentation blazing fast, simple, and data-centric</p><div class="flex flex-wrap items-center justify-center gap-3 text-sm text-muted-foreground"><div class="flex items-center gap-2"><span>Published</span><time dateTime="2025-12-23T04:10:27.099Z">December 23, 2025</time></div><span class="text-muted-foreground">•</span><span>8<!-- --> min read</span><span class="text-muted-foreground">•</span><a href="/instrumenting-at-10-years-per-second.md" target="_blank" rel="noopener" class="inline-flex items-center gap-2 text-muted-foreground transition-colors hover:text-foreground"><i class="fa-brands fa-markdown" aria-hidden="true"></i>View as Markdown</a></div></div></header></div><div class="w-full"><img alt="Instrumenting at 10 years per second" width="1200" height="750" decoding="async" data-nimg="1" class="w-full h-auto" style="color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg"/></div><div class="px-5 py-12 sm:px-8 space-y-8"><div class="max-w-2xl mx-auto"><div class="flex flex-wrap items-center gap-5"><div class="flex items-center gap-5"><div class="flex gap-3 items-start"><a target="_blank" rel="noopener" href="https://hashnode.com/@jaywhy13"><span class="relative flex shrink-0 overflow-hidden rounded-full h-10 w-10 border border-border transition-opacity hover:opacity-80"><span class="flex h-full w-full items-center justify-center rounded-full bg-muted text-muted-foreground font-semibold text-lg">J</span></span></a><div class="min-w-0"><div class="flex items-center gap-2"><a target="_blank" rel="noopener" class="font-medium text-foreground hover:text-primary transition-colors" href="https://hashnode.com/@jaywhy13">Jean-Mark Wright</a><div class="flex items-center gap-1.5 text-sm"><a target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground hover:text-foreground transition-colors" aria-label="X (Twitter)" href="https://twitter.com/kramnaej"><i class="fa-brands fa-x-twitter" aria-hidden="true"></i></a><a target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground hover:text-foreground transition-colors" aria-label="LinkedIn" href="https://www.linkedin.com/in/jean-mark-wright/"><i class="fa-brands fa-linkedin" aria-hidden="true"></i></a></div></div><div class="text-sm text-muted-foreground line-clamp-2"><p>Jean-Mark is an articulate, customer-focused, visionary with extensive industry experience encompassing front and back end development in all phases of software development. He lives "outside the box" in pursuit of architectural elegance and relevant, tangible solutions. He is a strong believer that team empowerment fosters genius and is versed at working solo.</p>
</div></div></div></div></div></div><div class="relative"><aside class="hidden xl:block absolute top-0 left-0 h-full"><nav class="sticky top-22 w-44"><div class="hidden xl:block group/toc relative"><div class="grid"><div class="[grid-area:1/1] opacity-0 group-hover/toc:opacity-100 transition-opacity duration-200 max-h-[calc(100vh-3rem)] overflow-y-auto"><p class="text-sm font-semibold uppercase tracking-wide text-muted-foreground mb-3">On this page</p><nav aria-label="On this page"><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-introduction">Introduction</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-solution-goals">Solution Goals</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-how-it-works">How it works</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0.75rem" href="#heading-the-configuration">The Configuration</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0.75rem" href="#heading-the-hook">The Hook</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0.75rem" href="#heading-coverage-report">Coverage Report</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-next-steps">Next Steps</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-conclusion">Conclusion</a></nav></div><div class="[grid-area:1/1] relative flex flex-col gap-px transition-opacity duration-200 group-hover/toc:opacity-0 pointer-events-none self-start max-h-[calc(100vh-3rem)] overflow-hidden pl-3"><div class="absolute left-0 w-0.5 rounded-full bg-foreground transition-all duration-300 opacity-0" style="transform:translateY(0px);height:12px"></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:12px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:14px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:23px"></div></div><div class="flex items-center h-3" style="padding-left:10px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:17px"></div></div><div class="flex items-center h-3" style="padding-left:10px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:10px"></div></div><div class="flex items-center h-3" style="padding-left:10px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:15px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:10px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:10px"></div></div></div></div></div></nav></aside><div class="max-w-2xl mx-auto min-w-0 contain-[inline-size]"><div class="prose dark:prose-invert prose-pre:bg-foreground prose-pre:text-background prose-pre:border prose-pre:border-border prose-pre:rounded prose-pre:px-5 prose-pre:py-4 dark:prose-pre:bg-muted dark:prose-pre:text-foreground dark:prose-pre:border-border prose-pre:overflow-x-auto max-w-none text-base sm:text-lg [&amp;&gt;div&gt;p:first-child]:mt-0 [&amp;&gt;div&gt;p:first-child]:pt-0 min-w-0 wrap-break-word [&amp;_a]:break-all **:max-w-full"><div><h2 id="heading-introduction">Introduction</h2>
<p>I've found a new love - custom instrumentation, and I can’t keep it to myself. We manually increased the custom instrumentation in one of our newer services and loved the impact. The instrumentation accelerated forensic investigations, root cause analysis and system comprehension. I’m hopelessly in love because I experienced worse. I’ve felt the pain of data insufficiency. We’ve had too many incidents where our traces could barely help us identify the impacted customer segment, much less tell us how many customers were impacted. Custom instrumentation has demonstrated its value, and we wanted to share it with the rest of the microservices in our fleet.</p>
<p>There was only one problem. Although we had one well-instrumented microservice, we still had over 40 other microservices that needed to be instrumented. Adding high-quality custom instrumentation takes time. Assuming it’d take about 3 months to instrument each of those services, it’d take ~10 years to get to the quality we needed. And which engineering leadership team doesn’t like to hear a feature which takes only 10 years to build? So… yah… we sought to fix that problem.</p>
<p>The rest of the talk covers a new approach we created called Config-Accelerated Custom Instrumentation, which we’re using to accelerate the time taken to add high-quality custom instrumentation to our services.</p>
<h2 id="heading-solution-goals">Solution Goals</h2>
<p>We had a few goals in mind for building an automated solution.</p>
<ul>
<li><p><strong>Speed</strong> - We want instrumentation creation to be really fast too. We had ~40+ services in our fleet that needed this tooling, and 10 years wasn’t going to cut it.</p>
</li>
<li><p><strong>Simplicity</strong> - we wanted a solution that enabled engineers to add instrumentation easily. Relative to the manual effort, it should be a piece of cake to get quality instrumentation captured. We want engineers to enjoy adding telemetry.</p>
</li>
<li><p><strong>Data-centric</strong> - we want our engineers to think simply about the data they want to capture. We’d do the heavy lifting and capture the data for them. So, we want them to think “these are the data points I want to capture”. We’d provide the necessary automation and logic to observe the application stack and collect the data.</p>
</li>
<li><p><strong>Small footprint</strong> - we also wanted a super unobtrusive implementation. We don’t want our tooling littering the codebase. A large footprint puts a real damper on engineer productivity and any evolutions we want to make.</p>
</li>
<li><p><strong>Verifiability</strong> - we want engineers to verify that the instrumentation works as expected. Our desire for simplicity would likely remove some visibility, so we want to restore that visibility with verifiable mechanisms. We needed to answer questions like “will this datapoint be captured in my telemetry?" or “which areas of my app are under-instrumented?”</p>
</li>
<li><p><strong>Secure</strong> - we don't want to risk any PII leakages. If we’re automating custom instrumentation, we’ll likely touch some PII and we’ll need to be careful nothing’s unintentionally leaked.</p>
</li>
</ul>
<h2 id="heading-how-it-works">How it works</h2>
<p>Our Config-Accelerated Custom Instrumentation (CACI) approach consisted of three distinct pieces. There’s the configuration that tells us what to instrument, a hook (metaclass) that captures the telemetry from classes and a report that provides feedback on what’ll be captured. Altogether, this simple set of tools helps us accelerate the pace of custom instrumentation dramatically!</p>
<h3 id="heading-the-configuration">The Configuration</h3>
<p>Let’s talk about the Configuration first. The configuration provides guidelines for what our tooling should capture for engineers. This is where engineers tell us what they want to instrument. The config captures allowed field names, field types, data classes and dictionaries. It functions as a purely opt-in configuration. Data has to be explicitly included; nothing is captured by default. Here’s what a sample configuration looks like:</p>
<pre><code class="lang-python">capture_configuration = TelemetryCaptureConfiguration(
allowed_field_names=[
<span class="hljs-string">"account_id"</span>,
<span class="hljs-string">"business_id"</span>,
],
disallowed_field_names=[],
allowed_field_types=[UUID, datetime, bool, Enum],
disallowed_field_types=[],
allowed_dataclasses=[
<span class="hljs-comment"># Common</span>
AllowedDataclass(
dataclass_name=<span class="hljs-string">"Business"</span>,
allowed_field_names=[
<span class="hljs-string">"business_id"</span>,
...
],
),
],
)
</code></pre>
<p>The configuration places the focus squarely on the application’s data. We want the configuration to be completely decoupled from the application’s architecture. This avoids fragility and is change-resistant. We don’t want engineers having to update their configuration every time they add a new function, or change an existing one. The configuration is deliberately simple, easy to maintain and extend. Using this, engineers can tell us what application data is important for us to capture. Once this configuration exists, we can collect the data with our hooks!</p>
<h3 id="heading-the-hook">The Hook</h3>
<p>Once we’ve got the configuration telling us what needs to be captured, now we need a mechanism for collecting the data exchanged between components in the application stack. We want to capture information exchanged by components. We started by capturing the function inputs for the component. So, effectively, every time some component calls another component, we want to inspect the function input to see if any of the variables are specified in the configuration. To enable this, we created a metaclass that wraps each public method, captures relevant inputs then passes through outputs and errors. We only want to do this for public methods to keep things simple and easy to understand. We opted to use a metaclass as our hook. Using a metaclass, we can hook into the class creation and refine the methods dynamically, inserting our logic.</p>
<p>The basic logic looks like this…</p>
<pre><code class="lang-python"><span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">ComponentMetaclass</span>(<span class="hljs-params">type</span>):</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__new__</span>(<span class="hljs-params">...</span>):</span>
<span class="hljs-comment"># Loop over all methods</span>
<span class="hljs-keyword">for</span> method_name, method <span class="hljs-keyword">in</span> methods:
<span class="hljs-comment"># Wrap the method with our logic</span>
new_method = wrap_method_with_our_logic(method)
<span class="hljs-comment"># Replace the method definition</span>
attrs[method_name] = new_method
</code></pre>
<p>Then, using it is really easy. We just specify the metaclass on any class to use it.</p>
<pre><code class="lang-python"><span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">SubscriptionService</span>(<span class="hljs-params">metaclass=ComponentMetaclass</span>):</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">create_subscription</span>(<span class="hljs-params">self, business_id: BusinessId, user_id: UserId</span>):</span>
...
</code></pre>
<p>And that’s it! Just by specifying that a component class wants to use our metaclass, our Observability package gets to work! It starts intercepting all calls. For instance, when a call is made to <code>SubscriptionService().create_subscription(….)</code> it calls our logic and checks if the <code>business_id</code> and <code>user_id</code> are in the configuration (<code>TelemetryCaptureConfiguration</code>). If it is, we capture those inputs for submission later.</p>
<h3 id="heading-coverage-report">Coverage Report</h3>
<p>An important aspect of our design was to provide an explicit feedback loop engineers could use. These tools enable engineers to cover more ground faster, however, it also removes considerable visibility for them. Previously, an engineer could look at a method and see what’s instrumented. In the example below you can see where engineers would manually stuff data into a <code>ctx</code>. It was a tremendous feedback loop. Engineers can see the code and touch it. They can write test assertions to ensure it works as expected. By adding a configuration and these hooks, we’re granting power but removing visibility.</p>
<pre><code class="lang-python"><span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">SubscriptionService</span>:</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">create_subscription</span>(<span class="hljs-params">self, ctx, business: BusinessId, user_id: UserId</span>):</span>
<span class="hljs-comment"># Capture some telemetry</span>
ctx[<span class="hljs-string">"business_id"</span>] = business_id
ctx[<span class="hljs-string">"user_id"</span>] = user_id
<span class="hljs-comment"># do actual logic ...</span>
</code></pre>
<p>To replace this visibility, we provide engineers with a Coverage Report. It’s a report that generates a method-by-method commentary on what will or won’t get captured by our tooling. It also provides a percentage of how many variables are captured.</p>
<pre><code class="lang-plaintext">- SubscriptionService
- create_subscription(self, business_id: BusinessId, user_id: UserId)
- Inputs
- business_id:BusinessId - ✅
- user_id:UserId - 🚫
-----------------------------------------------------------------
Parameters captured: 1/2 (50%)
</code></pre>
<p>This restores visibility for engineers aiming to understand what’s instrumented and what’s not. It also provides a goal post they can aim for. Currently, we encourage engineers to aim for 70% or higher, but it’s not enforced currently.</p>
<h2 id="heading-next-steps">Next Steps</h2>
<p>There are a few improvements we’re thinking of making. We want to decouple it from specific object types to simplify things even further. Engineers are less likely to write format-specific rules for the same piece of data. Our configuration has explicit support for data classes and Python dictionaries. However, I’ve thought of using it in other places where we’re using different data types but handling the same data. I want to transition it to where it thinks generically about primitive and complex object types (including nested ones), regardless of the format.</p>
<p>We also want to move beyond coverage. Coverage is a controversial metric in engineering. It may improve code quality but aiming too high can be pointless, arbitrary and force engineers to game the system. Coverage gives us a basic idea of what we’ll capture, but it falls short of the real picture. We want to know if the data will be captured in a particular workflow. It doesn’t matter that the data exists at a certain level. We’ll definitely want to extend this model so it’s more sophisticated.</p>
<p>Finally, we want to capture function outputs too. I haven’t figured a great way to represent outputs given they’re unnamed in Python. I don’t want to couple the naming to the structure. Eventually we’ll figure that out.</p>
<h2 id="heading-conclusion">Conclusion</h2>
<p>Config-Accelerated Custom Instrumentation has been a game-changer for us. What started as a 10-year problem became something we could tackle in a reasonable timeframe. A configuration file, a metaclass, and a coverage report—and that's actually it!</p>
<p>We're still iterating and improving, but even in its current form, CACI has proven its worth. Our services are better instrumented, our incidents are easier to debug, and our engineers are spending less time manually adding telemetry and more time building features.</p>
<p>If you're staring down a similar mountain of under-instrumented services, I'd encourage you to think about what automation could do for you. The manual approach works, but it doesn't scale. And in a world where observability can make or break your ability to respond to incidents, waiting 10 years isn't an option.</p>
<p>Thanks for stopping by!</p>
</div></div></div></div><div class="max-w-2xl mx-auto border-t border-border pt-8 space-y-5"><div class="flex flex-wrap items-center gap-2"><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/observability">#<!-- -->observability</a><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/instrumentation">#<!-- -->instrumentation</a><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/python">#<!-- -->python</a><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/python3">#<!-- -->python3</a></div></div></div></article></div></div></div></section><section class="pb-8"><div class="px-4 sm:px-8"><div class="w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="max-w-4xl w-full mx-auto"><div class="rounded-xl border bg-card text-card-foreground p-5 sm:p-8"><section class="space-y-5"><div class="flex items-center justify-between"><h2 class="text-lg font-bold tracking-tight">Comments</h2><a target="_blank" rel="noopener" class="inline-flex items-center gap-2 rounded-md border border-border px-3 py-1.5 text-sm font-medium text-foreground transition hover:border-primary hover:text-primary" href="https://hashnode.com/posts/instrumenting-at-10-years-per-second/694a1633103f879a13892085"><i class="fa-regular fa-comment" aria-hidden="true"></i>Join the discussion</a></div><p class="text-sm text-muted-foreground py-5 text-center">No comments yet.<!-- --> <a target="_blank" rel="noopener" class="text-primary hover:underline" href="https://hashnode.com/posts/instrumenting-at-10-years-per-second/694a1633103f879a13892085">Be the first to comment.</a></p></section></div></div></div></div></section><section class="pb-8"><div class="px-4 sm:px-8"><div class="w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="max-w-4xl w-full mx-auto"><section class="space-y-6"><h2 class="text-2xl font-bold tracking-tight text-balance">More from this blog</h2><div class="flex flex-col gap-6"><article aria-labelledby="_R_panpfalb_" class="h-full"><a class="block h-full" href="/on-shipping-smaller-prs"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_panpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">On shipping smaller PRs</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">Concrete examples for how to ship smaller PRs</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Feb 17, 2025</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>8 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="On shipping smaller PRs" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/o9oQaOGpLz0/upload/9e19fbfd378d65c448f7233b2a273b79.jpeg"/></div></div></div></div></a></article><article aria-labelledby="_R_19anpfalb_" class="h-full"><a class="block h-full" href="/small-prs-ship-faster-qed"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_19anpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Small PRs ship faster - QED</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">My unrestrained passion for shipping smaller PRs constrained into a handful of benefits</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Dec 25, 2024</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>5 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Small PRs ship faster - QED" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/JzMS3RLSZn8/upload/a18ee8f82d0d56644ab82f6c1bead8b4.jpeg"/></div></div></div></div></a></article><article aria-labelledby="_R_1panpfalb_" class="h-full"><a class="block h-full" href="/order-matters-making-a-compound-index-50x-faster"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_1panpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Order matters - making a compound index 50x faster</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">A story of how we made a compound index 50x faster by re-organizing the fields</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Oct 17, 2024</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>6 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Order matters - making a compound index 50x faster" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/upload/v1726107899370/04b1ac6d-7825-4a05-b5d0-0be78a2f49d9.png"/></div></div></div></div></a></article><article aria-labelledby="_R_29anpfalb_" class="h-full"><a class="block h-full" href="/dear-editor-we-need-better-database-observability"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_29anpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Dear Editor: We need better Database Observability</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">A call for better database observability, or awareness to a better reality</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Sep 12, 2024</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>8 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Dear Editor: We need better Database Observability" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/NvFkYV2ngOk/upload/2f4cbc39f32355a334d9119af2dd086c.jpeg"/></div></div></div></div></a></article><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 p-5 flex flex-col"><div class="flex flex-col items-center text-center gap-5"><div class="flex flex-col items-center gap-3"><div class="flex h-16 w-16 shrink-0 items-center justify-center rounded-full bg-muted"><i class="text-2xl fa-brands fa-hashnode text-foreground" aria-hidden="true"></i></div><div class="space-y-1"><p class="text-base font-semibold text-foreground leading-tight">Perspective Unspoken</p><p class="text-sm text-muted-foreground">19<!-- --> posts</p></div></div><div class="prose prose-sm dark:prose-invert max-w-xs mx-auto text-muted-foreground"><p>Welcome to "Perspective Unspoken", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.</p>
</div><div class="flex flex-wrap items-center justify-center gap-3"><a href="https://twitter.com/kramnaej" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="Twitter"><i class="fa-brands fa-x-twitter" aria-hidden="true"></i></a><a href="https://github.com/jaywhy13" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="GitHub"><i class="fa-brands fa-github" aria-hidden="true"></i></a><a href="https://www.linkedin.com/in/jean-mark-wright/" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="LinkedIn"><i class="fa-brands fa-linkedin" aria-hidden="true"></i></a><a href="https://hashnode.com/@jaywhy13" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="Hashnode"><i class="fa-brands fa-hashnode" aria-hidden="true"></i></a></div></div></div></div></section></div></div></div></section><script type="application/ld+json">{"@context":"https://schema.org","@type":"Article","url":"https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second","mainEntityOfPage":"https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second","headline":"Instrumenting at 10 years per second","description":"Introduction\nI've found a new love - custom instrumentation, and I can’t keep it to myself. We manually increased the custom instrumentation in one of our newer services and loved the impact. The instrumentation accelerated forensic investigations, r...","author":{"@type":"Person","name":"Jean-Mark Wright","url":"https://hashnode.com/@jaywhy13","image":"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg","sameAs":"https://x.com/kramnaej"},"publisher":{"@type":"Organization","name":"Perspective Unspoken","url":"https://jaywhy13.hashnode.dev"},"datePublished":"2025-12-23T04:10:27.099Z","dateModified":"2025-12-23T04:10:27.099Z","image":{"@type":"ImageObject","url":"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg"},"keywords":"observability, instrumentation, Python, Python 3"}</script><footer class="px-4 pt-8 pb-20 sm:px-8 text-muted-foreground" aria-label="Footer"><div class="w-full max-w-4xl mx-auto"><div class="flex flex-col items-center gap-4 text-center text-sm"><p class="font-medium"><time dateTime="2026"><span class="font-sans">©</span> <!-- -->2026</time> <!-- -->Perspective Unspoken</p><nav aria-label="Footer navigation"><ul class="flex flex-wrap justify-center gap-4 text-sm tracking-wide"><li><a class="transition hover:text-foreground" href="/archive">Archive</a></li><li><a class="transition hover:text-foreground" target="_blank" rel="noopener" href="https://hashnode.com/privacy-policy">Privacy</a></li><li><a class="transition hover:text-foreground" target="_blank" rel="noopener" href="https://hashnode.com/terms">Terms</a></li></ul></nav><div class="pt-8 flex justify-center gap-6"><a target="_blank" rel="noopener" class="flex items-center gap-2 text-sm transition hover:text-foreground" title="Sitemap" href="/sitemap.xml"><i class="fa-regular fa-sitemap"></i><span>Sitemap</span></a><a target="_blank" rel="noopener" class="flex items-center gap-2 text-sm transition hover:text-foreground" title="RSS Feed" href="/rss.xml"><i class="fa-regular fa-rss"></i><span>RSS</span></a></div><div class="pt-10 text-center"><a target="_blank" rel="noopener" aria-label="Visit Hashnode (opens in a new tab)" class="inline-flex items-center justify-center gap-2 whitespace-nowrap font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent hover:text-accent-foreground h-10 rounded-md text-xl px-5 py-4" href="https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev&amp;utm_medium=referral&amp;utm_campaign=blog_footer_logo&amp;utm_content=logo"><svg fill="none" viewBox="0 0 282 39" aria-hidden="true" class="h-6 w-auto min-w-[9.5rem] text-muted-foreground transition hover:text-foreground"><path fill="currentColor" d="M2.68 13.032c-3.573 3.505-3.573 9.363 0 12.936L13.032 36.32c3.505 3.573 9.363 3.573 12.936 0L36.32 25.968c3.573-3.573 3.573-9.431 0-12.936L25.968 2.68c-3.573-3.573-9.431-3.573-12.936 0zm12.211 1.935c2.507-2.521 6.582-2.544 9.104-.038s2.544 6.582.038 9.104-6.582 2.544-9.104.038-2.544-6.582-.038-9.104"></path><path fill="currentColor" d="M268.379 39q-6.322 0-10.35-4.108-4.002-4.134-4.002-10.483 0-6.296 4.002-10.43 4-4.135 10.163-4.135 6.028 0 9.817 3.734 3.814 3.735 3.814 9.497v2.8h-24.248v-4.054h17.633q-.187-2.747-2.054-4.508-1.868-1.76-4.962-1.76-3.28-.001-5.388 2.374-2.108 2.373-2.108 6.375 0 4.188 2.348 6.695 2.347 2.481 5.895 2.481 1.68 0 2.934-.453a8.5 8.5 0 0 0 2.321-1.307q1.041-.854 2.268-2.614l4.694 2.96q-1.707 2.748-3.494 4.135a12.7 12.7 0 0 1-3.975 2.081q-2.187.72-5.308.72m-33.496 0q-5.575 0-9.283-4.028-3.681-4.029-3.681-10.484 0-6.43 3.761-10.51 3.76-4.108 9.47-4.108 2.8 0 5.175 1.2 2.4 1.174 3.628 3.148l.026-.08V0h7.016v38.2h-6.722v-4.722l.64.8-.694.133q-1.28 2.108-3.788 3.361Q237.924 39 234.883 39m1.627-6.002q3.308 0 5.495-2.428t2.188-6.135q0-3.762-2.214-6.109-2.214-2.374-5.469-2.374-3.28 0-5.522 2.347-2.214 2.348-2.214 6.163 0 3.76 2.214 6.162 2.214 2.374 5.522 2.374M205.811 39q-6.376 0-10.537-4.161-4.161-4.188-4.161-10.404 0-6.321 4.161-10.456 4.188-4.135 10.537-4.135 6.348 0 10.537 4.161 4.188 4.161 4.188 10.43 0 6.216-4.188 10.404Q212.186 39 205.811 39m0-6.162q3.307 0 5.575-2.374 2.267-2.374 2.267-6.055 0-3.707-2.294-6.056-2.267-2.347-5.548-2.347-3.308 0-5.575 2.374-2.268 2.348-2.268 6.029t2.241 6.055q2.268 2.374 5.602 2.374M162.586 38.2V10.644h6.802v5.015l-.8-.987h.906q1.308-2.187 3.762-3.494 2.48-1.335 5.228-1.334 4.615 0 7.256 2.8 2.667 2.776 2.667 7.603V38.2h-7.069V21.794q0-2.854-1.387-4.401t-4.134-1.547q-2.614 0-4.402 1.974-1.76 1.974-1.76 4.881V38.2zm-30.181 0V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.069V21.794q0-2.827-1.387-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.814 2.027-1.814 4.962v15.391zm-14.416.8q-2.882 0-5.015-.64-2.108-.614-3.761-1.84-1.654-1.228-2.668-2.695l-.987-1.467 4.695-3.174.773 1.067a9.1 9.1 0 0 0 1.788 1.814 8.2 8.2 0 0 0 2.32 1.2q1.308.4 3.068.4 2.161 0 3.441-.8 1.308-.8 1.307-2.294 0-1.36-1.147-2.108-1.12-.747-5.068-1.654-5.335-1.253-7.656-3.334-2.32-2.107-2.321-5.362 0-3.76 3.148-6.002 3.148-2.267 7.923-2.267 2.454 0 4.348.507t3.388 1.493q1.493.988 2.454 2.188l.987 1.2-4.482 3.255-.667-.774a7.4 7.4 0 0 0-1.627-1.387 6.4 6.4 0 0 0-2.054-.88 9.5 9.5 0 0 0-2.401-.294q-1.974 0-3.201.8-1.2.774-1.2 1.921 0 1.254 1.12 2.027 1.147.748 5.229 1.681 5.334 1.227 7.629 3.308 2.294 2.055 2.294 5.308 0 4.029-3.228 6.43Q123.217 39 117.989 39M87.61 39q-4.508 0-7.335-2.454-2.828-2.481-2.828-6.482 0-4.188 3.254-6.669t8.483-2.48q2.082 0 4.028.426 1.974.427 3.361 1.12v-1.6q0-2.588-1.813-4.082-1.788-1.493-4.615-1.493-1.628 0-2.801.373a8.7 8.7 0 0 0-2.188.987q-.987.614-1.84 1.414l-.854.8-4.081-3.788 1.307-1.173a18.7 18.7 0 0 1 2.934-2.108q1.601-.933 3.601-1.44t4.589-.507q5.868 0 9.229 2.88 3.361 2.882 3.361 8.084V38.2h-6.749v-5.922l1.654 2.187h-1.76q-1.308 2.055-3.655 3.308Q90.545 39 87.611 39m1.814-4.988q3.148 0 5.149-2.187 2-2.215 2-5.309v-.08q-1.12-.693-2.72-1.094a13 13 0 0 0-3.388-.426q-2.828 0-4.562 1.2-1.734 1.173-1.734 3.521 0 2.028 1.468 3.201t3.787 1.174M49 38.2V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.07V21.794q0-2.827-1.386-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.815 2.027-1.814 4.962v15.391z"></path></svg></a></div></div></div></footer></main><!--$--><!--/$--><script src="/_next/static/chunks/webpack-d716a75e4017380e.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[16174,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"ThemeProvider\"]\n3:I[82180,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"TooltipProvider\"]\n4:I[63851,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"UrqlProvider\"]\n5:I[82714,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"ConsentManager\"]\n6:I[9766,[],\"\"]\n7:I[98924,[],\"\"]\n8:I[33293,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"4345\",\"static/chunks/app/not-found-b861935655c5698b.js\"],\"default\"]\n9:I[21447,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"Integrations\"]\na:I[35514,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"Analytics\"]\nc:I[24431,[],\"OutletBoundary\"]\ne:I[15278,[],\"AsyncMetadataOutlet\"]\n10:I[24431,[],\"ViewportBoundary\"]\n12:I[24431,[],\"MetadataBoundary\"]\n13:\"$Sreact.suspense\"\n15:I[57150,[],\"\"]\n:HL[\"/_next/static/media/304eba2dc288a929-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/38077b822d9512d7-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/40ddcd2db5e33048-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/707228f0d4113e57-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/a73fce2732cf3a48-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/c25783985a269397-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/c86b497a7be35730-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/d9cd7618114896c5-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/f4e6f3e26e87864d-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/fde6a32e2c53c69f-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/css/bc5c0eaecd39bc7d.css\",\"style\"]\n:HL[\"/_next/static/css/7ffcc02a3c67389b.css\",\"style\"]\n:HL[\"/_next/static/css/c20e76e7d11dec32.css\",\"style\"]\n:HL[\"/_next/static/css/a6eb8c1f37be68d5.css\",\"style\"]\n0:{\"P\":null,\"b\":\"ePiaRevyKQ9kV7jt41IDP\",\"p\":\"\",\"c\":[\"\",\"instrumenting-at-10-years-per-second\"],\"i\":false,\"f\":[[[\"\",{\"children\":[[\"slug\",\"instrumenting-at-10-years-per-second\",\"c\"],{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],[\"\",[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/bc5c0eaecd39bc7d.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/7ffcc02a3c67389b.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\","])</script><script>self.__next_f.push([1,"\"link\",\"2\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/c20e76e7d11dec32.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"3\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/a6eb8c1f37be68d5.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"suppressHydrationWarning\":true,\"children\":[[\"$\",\"head\",null,{\"children\":[\"$\",\"script\",null,{\"type\":\"text/javascript\",\"dangerouslySetInnerHTML\":{\"__html\":\"window.addEventListener('message',function(e){var d=e.data;if(!d||d.__hnWidget!==1||typeof d.h!=='number'||d.h\u003c=0)return;var f=document.querySelectorAll('.hn-widget-frame');for(var i=0;i\u003cf.length;i++){if(f[i].contentWindow===e.source){f[i].style.height=d.h+'px';break;}}});\"}}]}],[\"$\",\"body\",null,{\"className\":\"__variable_ede2c9 __variable_558dcf bg-background dark:bg-background\",\"children\":[\"$\",\"$L2\",null,{\"attribute\":\"class\",\"defaultTheme\":\"system\",\"enableSystem\":true,\"disableTransitionOnChange\":true,\"children\":[\"$\",\"$L3\",null,{\"children\":[\"$\",\"$L4\",null,{\"children\":[\"$\",\"$L5\",null,{\"children\":[[\"$\",\"$L6\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L7\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"$L8\",null,{}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}],[\"$\",\"$L9\",null,{}],[\"$\",\"$La\",null,{}]]}]}]}]}]}]]}]]}],{\"children\":[[\"slug\",\"instrumenting-at-10-years-per-second\",\"c\"],[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L6\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L7\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[\"__PAGE__\",[\"$\",\"$1\",\"c\",{\"children\":[\"$Lb\",null,[\"$\",\"$Lc\",null,{\"children\":[\"$Ld\",[\"$\",\"$Le\",null,{\"promise\":\"$@f\"}]]}]]}],{},null,false]},null,false]},null,false],[\"$\",\"$1\",\"h\",{\"children\":[null,[[\"$\",\"$L10\",null,{\"children\":\"$L11\"}],[\"$\",\"meta\",null,{\"name\":\"next-size-adjust\",\"content\":\"\"}]],[\"$\",\"$L12\",null,{\"children\":[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$13\",null,{\"fallback\":null,\"children\":\"$L14\"}]}]}]]}],false]],\"m\":\"$undefined\",\"G\":[\"$15\",[]],\"s\":false,\"S\":false}\n11:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}]]\nd:null\n16:I[80622,[],\"IconMark\"]\nf:{\"metadata\":[[\"$\",\"title\",\"0\",{\"children\":\"Accelerating the pace of custom instrumentation\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"Struggling to instrument dozens of microservices? Discover our config-driven approach that accelerates custom instrumentation from years to weeks.\"}],[\"$\",\"link\",\"2\",{\"rel\":\"canonical\",\"href\":\"https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:title\",\"content\":\"Accelerating the pace of custom instrumentation\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:description\",\"content\":\"Struggling to instrument dozens of microservices? Discover our config-driven approach that accelerates custom instrumentation from years to weeks.\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:url\",\"content\":\"https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:site_name\",\"content\":\"Perspective Unspoken\"}],[\"$\",\"meta\",\"7\",{\"property\":\"og:image\",\"content\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg\"}],[\"$\",\"meta\",\"8\",{\"property\":\"og:image:width\",\"content\":\"1200\"}],[\"$\",\"meta\",\"9\",{\"property\":\"og:image:height\",\"content\":\"630\"}],[\"$\",\"meta\",\"10\",{\"property\":\"og:type\",\"content\":\"article\"}],[\"$\",\"meta\",\"11\",{\"property\":\"article:published_time\",\"content\":\"2025-12-23T04:10:27.099Z\"}],[\"$\",\"meta\",\"12\",{\"property\":\"article:author\",\"content\":\"https://jaywhy13.hashnode.dev/@jaywhy13\"}],[\"$\",\"meta\",\"13\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"14\",{\"name\":\"twitter:creator\",\"cont"])</script><script>self.__next_f.push([1,"ent\":\"https://twitter.com/kramnaej\"}],[\"$\",\"meta\",\"15\",{\"name\":\"twitter:title\",\"content\":\"Accelerating the pace of custom instrumentation\"}],[\"$\",\"meta\",\"16\",{\"name\":\"twitter:description\",\"content\":\"Struggling to instrument dozens of microservices? Discover our config-driven approach that accelerates custom instrumentation from years to weeks.\"}],[\"$\",\"meta\",\"17\",{\"name\":\"twitter:image\",\"content\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg\"}],[\"$\",\"link\",\"18\",{\"rel\":\"icon\",\"href\":\"/favicon.ico\"}],[\"$\",\"$L16\",\"19\",{}]],\"error\":null,\"digest\":\"$undefined\"}\n14:\"$f:metadata\"\n17:I[652,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"8974\",\"static/chunks/app/page-cc18ae1a3a7a5ed8.js\"],\"BlogHeader\"]\n18:T31b3,\u003ch2 id=\"heading-introduction\"\u003eIntroduction\u003c/h2\u003e\n\u003cp\u003eI've found a new love - custom instrumentation, and I can’t keep it to myself. We manually increased the custom instrumentation in one of our newer services and loved the impact. The instrumentation accelerated forensic investigations, root cause analysis and system comprehension. I’m hopelessly in love because I experienced worse. I’ve felt the pain of data insufficiency. We’ve had too many incidents where our traces could barely help us identify the impacted customer segment, much less tell us how many customers were impacted. Custom instrumentation has demonstrated its value, and we wanted to share it with the rest of the microservices in our fleet.\u003c/p\u003e\n\u003cp\u003eThere was only one problem. Although we had one well-instrumented microservice, we still had over 40 other microservices that needed to be instrumented. Adding high-quality custom instrumentation takes time. Assuming it’d take about 3 months to instrument each of those services, it’d take ~10 years to get to the quality we needed. And which engineering leadership team doesn’t like to hear a feature which takes only 10 years to build? So… yah… we sought to fix that problem.\u003c/p\u003e\n\u003cp\u003eThe rest of the talk covers a new approach we created called Config-Accelerated Custom Instrumentation, which we’re using to accelerate the time taken to add high-quality custom instrumentation to our services.\u003c/p\u003e\n\u003ch2 id=\"heading-solution-goals\"\u003eSolution Goals\u003c/h2\u003e\n\u003cp\u003eWe had a few goals in mind for building an automated solution.\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eSpeed\u003c/strong\u003e - We want instrumentation creation to be really fast too. We had ~40+ services in our fleet that needed this tooling, and 10 years wasn’t going to cut it.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eSimplicity\u003c/strong\u003e - we wanted a solution that enabled engineers to add instrumentation easily. Relative to the manual effort, it should be a piece of cake to get quality instrumentation captured. We want engineers to enjoy adding telemetry.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eData-centric\u003c/strong\u003e - we want our engineers to think simply about the data they want to capture. We’d do the heavy lifting and capture the data for them. So, we want them to think “these are the data points I want to capture”. We’d provide the necessary automation and logic to observe the application stack and collect the data.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eSmall footprint\u003c/strong\u003e - we also wanted a super unobtrusive implementation. We don’t want our tooling littering the codebase. A large footprint puts a real damper on engineer productivity and any evolutions we want to make.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eVerifiability\u003c/strong\u003e - we want engineers to verify that the instrumentation works as expected. Our desire for simplicity would likely remove some visibility, so we want to restore that visibility with verifiable mechanisms. We needed to answer questions like “will this datapoint be captured in my telemetry?\" or"])</script><script>self.__next_f.push([1," “which areas of my app are under-instrumented?”\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eSecure\u003c/strong\u003e - we don't want to risk any PII leakages. If we’re automating custom instrumentation, we’ll likely touch some PII and we’ll need to be careful nothing’s unintentionally leaked.\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003ch2 id=\"heading-how-it-works\"\u003eHow it works\u003c/h2\u003e\n\u003cp\u003eOur Config-Accelerated Custom Instrumentation (CACI) approach consisted of three distinct pieces. There’s the configuration that tells us what to instrument, a hook (metaclass) that captures the telemetry from classes and a report that provides feedback on what’ll be captured. Altogether, this simple set of tools helps us accelerate the pace of custom instrumentation dramatically!\u003c/p\u003e\n\u003ch3 id=\"heading-the-configuration\"\u003eThe Configuration\u003c/h3\u003e\n\u003cp\u003eLet’s talk about the Configuration first. The configuration provides guidelines for what our tooling should capture for engineers. This is where engineers tell us what they want to instrument. The config captures allowed field names, field types, data classes and dictionaries. It functions as a purely opt-in configuration. Data has to be explicitly included; nothing is captured by default. Here’s what a sample configuration looks like:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003ecapture_configuration = TelemetryCaptureConfiguration(\n allowed_field_names=[\n \u003cspan class=\"hljs-string\"\u003e\"account_id\"\u003c/span\u003e,\n \u003cspan class=\"hljs-string\"\u003e\"business_id\"\u003c/span\u003e,\n ],\n disallowed_field_names=[],\n allowed_field_types=[UUID, datetime, bool, Enum],\n disallowed_field_types=[],\n allowed_dataclasses=[\n \u003cspan class=\"hljs-comment\"\u003e# Common\u003c/span\u003e\n AllowedDataclass(\n dataclass_name=\u003cspan class=\"hljs-string\"\u003e\"Business\"\u003c/span\u003e,\n allowed_field_names=[\n \u003cspan class=\"hljs-string\"\u003e\"business_id\"\u003c/span\u003e,\n ...\n ],\n ),\n ],\n)\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThe configuration places the focus squarely on the application’s data. We want the configuration to be completely decoupled from the application’s architecture. This avoids fragility and is change-resistant. We don’t want engineers having to update their configuration every time they add a new function, or change an existing one. The configuration is deliberately simple, easy to maintain and extend. Using this, engineers can tell us what application data is important for us to capture. Once this configuration exists, we can collect the data with our hooks!\u003c/p\u003e\n\u003ch3 id=\"heading-the-hook\"\u003eThe Hook\u003c/h3\u003e\n\u003cp\u003eOnce we’ve got the configuration telling us what needs to be captured, now we need a mechanism for collecting the data exchanged between components in the application stack. We want to capture information exchanged by components. We started by capturing the function inputs for the component. So, effectively, every time some component calls another component, we want to inspect the function input to see if any of the variables are specified in the configuration. To enable this, we created a metaclass that wraps each public method, captures relevant inputs then passes through outputs and errors. We only want to do this for public methods to keep things simple and easy to understand. We opted to use a metaclass as our hook. Using a metaclass, we can hook into the class creation and refine the methods dynamically, inserting our logic.\u003c/p\u003e\n\u003cp\u003eThe basic logic looks like this…\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eComponentMetaclass\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003etype\u003c/span\u003e):\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003e__new__\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e...\u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Loop over all methods\u003c/span\u003e\n \u003cspan class=\"hljs-keyword\"\u003efor\u003c/span\u003e method_name, method \u003cspan class=\"hljs-keyword\"\u003ein\u003c/span\u003e methods:\n \u003cspan class=\"hljs-comment\"\u003e# Wrap the method with our logic\u003c/span\u003e\n new_method = wrap_method_with_o"])</script><script>self.__next_f.push([1,"ur_logic(method)\n \u003cspan class=\"hljs-comment\"\u003e# Replace the method definition\u003c/span\u003e\n attrs[method_name] = new_method\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThen, using it is really easy. We just specify the metaclass on any class to use it.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionService\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003emetaclass=ComponentMetaclass\u003c/span\u003e):\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003eself, business_id: BusinessId, user_id: UserId\u003c/span\u003e):\u003c/span\u003e\n ...\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eAnd that’s it! Just by specifying that a component class wants to use our metaclass, our Observability package gets to work! It starts intercepting all calls. For instance, when a call is made to \u003ccode\u003eSubscriptionService().create_subscription(….)\u003c/code\u003e it calls our logic and checks if the \u003ccode\u003ebusiness_id\u003c/code\u003e and \u003ccode\u003euser_id\u003c/code\u003e are in the configuration (\u003ccode\u003eTelemetryCaptureConfiguration\u003c/code\u003e). If it is, we capture those inputs for submission later.\u003c/p\u003e\n\u003ch3 id=\"heading-coverage-report\"\u003eCoverage Report\u003c/h3\u003e\n\u003cp\u003eAn important aspect of our design was to provide an explicit feedback loop engineers could use. These tools enable engineers to cover more ground faster, however, it also removes considerable visibility for them. Previously, an engineer could look at a method and see what’s instrumented. In the example below you can see where engineers would manually stuff data into a \u003ccode\u003ectx\u003c/code\u003e. It was a tremendous feedback loop. Engineers can see the code and touch it. They can write test assertions to ensure it works as expected. By adding a configuration and these hooks, we’re granting power but removing visibility.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionService\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003eself, ctx, business: BusinessId, user_id: UserId\u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Capture some telemetry\u003c/span\u003e\n ctx[\u003cspan class=\"hljs-string\"\u003e\"business_id\"\u003c/span\u003e] = business_id\n ctx[\u003cspan class=\"hljs-string\"\u003e\"user_id\"\u003c/span\u003e] = user_id\n\n \u003cspan class=\"hljs-comment\"\u003e# do actual logic ...\u003c/span\u003e\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eTo replace this visibility, we provide engineers with a Coverage Report. It’s a report that generates a method-by-method commentary on what will or won’t get captured by our tooling. It also provides a percentage of how many variables are captured.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-plaintext\"\u003e- SubscriptionService\n - create_subscription(self, business_id: BusinessId, user_id: UserId)\n - Inputs\n - business_id:BusinessId - ✅\n - user_id:UserId - 🚫\n\n-----------------------------------------------------------------\nParameters captured: 1/2 (50%)\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThis restores visibility for engineers aiming to understand what’s instrumented and what’s not. It also provides a goal post they can aim for. Currently, we encourage engineers to aim for 70% or higher, but it’s not enforced currently.\u003c/p\u003e\n\u003ch2 id=\"heading-next-steps\"\u003eNext Steps\u003c/h2\u003e\n\u003cp\u003eThere are a few improvements we’re thinking of making. We want to decouple it from specific object types to simplify things even further. Engineers are less likely to write format-specific rules for the same piece of data. Our configuration has explicit support for data classes and Python dictionaries. However, I’ve thought of using it in other places where we’re using different data types but handling the same data. I want to transition it to where it thinks generically about primitive and complex object types (including nested ones), regardless of the format.\u003c/p\u003e\n\u003cp\u003eWe also want to move beyond coverage. Coverage is a controversial metric in engineering. It may improve code quality but aimi"])</script><script>self.__next_f.push([1,"ng too high can be pointless, arbitrary and force engineers to game the system. Coverage gives us a basic idea of what we’ll capture, but it falls short of the real picture. We want to know if the data will be captured in a particular workflow. It doesn’t matter that the data exists at a certain level. We’ll definitely want to extend this model so it’s more sophisticated.\u003c/p\u003e\n\u003cp\u003eFinally, we want to capture function outputs too. I haven’t figured a great way to represent outputs given they’re unnamed in Python. I don’t want to couple the naming to the structure. Eventually we’ll figure that out.\u003c/p\u003e\n\u003ch2 id=\"heading-conclusion\"\u003eConclusion\u003c/h2\u003e\n\u003cp\u003eConfig-Accelerated Custom Instrumentation has been a game-changer for us. What started as a 10-year problem became something we could tackle in a reasonable timeframe. A configuration file, a metaclass, and a coverage report—and that's actually it!\u003c/p\u003e\n\u003cp\u003eWe're still iterating and improving, but even in its current form, CACI has proven its worth. Our services are better instrumented, our incidents are easier to debug, and our engineers are spending less time manually adding telemetry and more time building features.\u003c/p\u003e\n\u003cp\u003eIf you're staring down a similar mountain of under-instrumented services, I'd encourage you to think about what automation could do for you. The manual approach works, but it doesn't scale. And in a world where observability can make or break your ability to respond to incidents, waiting 10 years isn't an option.\u003c/p\u003e\n\u003cp\u003eThanks for stopping by!\u003c/p\u003e\n19:T2a0b,## Introduction\n\nI've found a new love - custom instrumentation, and I can’t keep it to myself. We manually increased the custom instrumentation in one of our newer services and loved the impact. The instrumentation accelerated forensic investigations, root cause analysis and system comprehension. I’m hopelessly in love because I experienced worse. I’ve felt the pain of data insufficiency. We’ve had too many incidents where our traces could barely help us identify the impacted customer segment, much less tell us how many customers were impacted. Custom instrumentation has demonstrated its value, and we wanted to share it with the rest of the microservices in our fleet.\n\nThere was only one problem. Although we had one well-instrumented microservice, we still had over 40 other microservices that needed to be instrumented. Adding high-quality custom instrumentation takes time. Assuming it’d take about 3 months to instrument each of those services, it’d take ~10 years to get to the quality we needed. And which engineering leadership team doesn’t like to hear a feature which takes only 10 years to build? So… yah… we sought to fix that problem.\n\nThe rest of the talk covers a new approach we created called Config-Accelerated Custom Instrumentation, which we’re using to accelerate the time taken to add high-quality custom instrumentation to our services.\n\n## Solution Goals\n\nWe had a few goals in mind for building an automated solution.\n\n* **Speed** - We want instrumentation creation to be really fast too. We had ~40+ services in our fleet that needed this tooling, and 10 years wasn’t going to cut it.\n \n* **Simplicity** - we wanted a solution that enabled engineers to add instrumentation easily. Relative to the manual effort, it should be a piece of cake to get quality instrumentation captured. We want engineers to enjoy adding telemetry.\n \n* **Data-centric** - we want our engineers to think simply about the data they want to capture. We’d do the heavy lifting and capture the data for them. So, we want them to think “these are the data points I want to capture”. We’d provide the necessary automation and logic to observe the application stack and collect the data.\n \n* **Small footprint** - we also wanted a super unobtrusive implementation. We don’t want our tooling littering the codebase. A large footprint puts a real damper on engineer productivity and any evolutions we want to make.\n \n* **Verifiability** - we want engineers to verify that the instrumentati"])</script><script>self.__next_f.push([1,"on works as expected. Our desire for simplicity would likely remove some visibility, so we want to restore that visibility with verifiable mechanisms. We needed to answer questions like “will this datapoint be captured in my telemetry?\" or “which areas of my app are under-instrumented?”\n \n* **Secure** - we don't want to risk any PII leakages. If we’re automating custom instrumentation, we’ll likely touch some PII and we’ll need to be careful nothing’s unintentionally leaked.\n \n\n## How it works\n\nOur Config-Accelerated Custom Instrumentation (CACI) approach consisted of three distinct pieces. There’s the configuration that tells us what to instrument, a hook (metaclass) that captures the telemetry from classes and a report that provides feedback on what’ll be captured. Altogether, this simple set of tools helps us accelerate the pace of custom instrumentation dramatically!\n\n### The Configuration\n\nLet’s talk about the Configuration first. The configuration provides guidelines for what our tooling should capture for engineers. This is where engineers tell us what they want to instrument. The config captures allowed field names, field types, data classes and dictionaries. It functions as a purely opt-in configuration. Data has to be explicitly included; nothing is captured by default. Here’s what a sample configuration looks like:\n\n```python\ncapture_configuration = TelemetryCaptureConfiguration(\n allowed_field_names=[\n \"account_id\",\n \"business_id\",\n ],\n disallowed_field_names=[],\n allowed_field_types=[UUID, datetime, bool, Enum],\n disallowed_field_types=[],\n allowed_dataclasses=[\n # Common\n AllowedDataclass(\n dataclass_name=\"Business\",\n allowed_field_names=[\n \"business_id\",\n ...\n ],\n ),\n ],\n)\n```\n\nThe configuration places the focus squarely on the application’s data. We want the configuration to be completely decoupled from the application’s architecture. This avoids fragility and is change-resistant. We don’t want engineers having to update their configuration every time they add a new function, or change an existing one. The configuration is deliberately simple, easy to maintain and extend. Using this, engineers can tell us what application data is important for us to capture. Once this configuration exists, we can collect the data with our hooks!\n\n### The Hook\n\nOnce we’ve got the configuration telling us what needs to be captured, now we need a mechanism for collecting the data exchanged between components in the application stack. We want to capture information exchanged by components. We started by capturing the function inputs for the component. So, effectively, every time some component calls another component, we want to inspect the function input to see if any of the variables are specified in the configuration. To enable this, we created a metaclass that wraps each public method, captures relevant inputs then passes through outputs and errors. We only want to do this for public methods to keep things simple and easy to understand. We opted to use a metaclass as our hook. Using a metaclass, we can hook into the class creation and refine the methods dynamically, inserting our logic.\n\nThe basic logic looks like this…\n\n```python\nclass ComponentMetaclass(type):\n\n def __new__(...):\n # Loop over all methods\n for method_name, method in methods:\n # Wrap the method with our logic\n new_method = wrap_method_with_our_logic(method)\n # Replace the method definition\n attrs[method_name] = new_method\n```\n\nThen, using it is really easy. We just specify the metaclass on any class to use it.\n\n```python\nclass SubscriptionService(metaclass=ComponentMetaclass):\n\n def create_subscription(self, business_id: BusinessId, user_id: UserId):\n ...\n```\n\nAnd that’s it! Just by specifying that a component class wants to use our metaclass, our Observability package gets to work! It starts intercepting all calls. For instance,"])</script><script>self.__next_f.push([1," when a call is made to `SubscriptionService().create_subscription(….)` it calls our logic and checks if the `business_id` and `user_id` are in the configuration (`TelemetryCaptureConfiguration`). If it is, we capture those inputs for submission later.\n\n### Coverage Report\n\nAn important aspect of our design was to provide an explicit feedback loop engineers could use. These tools enable engineers to cover more ground faster, however, it also removes considerable visibility for them. Previously, an engineer could look at a method and see what’s instrumented. In the example below you can see where engineers would manually stuff data into a `ctx`. It was a tremendous feedback loop. Engineers can see the code and touch it. They can write test assertions to ensure it works as expected. By adding a configuration and these hooks, we’re granting power but removing visibility.\n\n```python\nclass SubscriptionService:\n\n def create_subscription(self, ctx, business: BusinessId, user_id: UserId):\n # Capture some telemetry\n ctx[\"business_id\"] = business_id\n ctx[\"user_id\"] = user_id\n\n # do actual logic ...\n```\n\nTo replace this visibility, we provide engineers with a Coverage Report. It’s a report that generates a method-by-method commentary on what will or won’t get captured by our tooling. It also provides a percentage of how many variables are captured.\n\n```plaintext\n- SubscriptionService\n - create_subscription(self, business_id: BusinessId, user_id: UserId)\n - Inputs\n - business_id:BusinessId - ✅\n - user_id:UserId - 🚫\n\n-----------------------------------------------------------------\nParameters captured: 1/2 (50%)\n```\n\nThis restores visibility for engineers aiming to understand what’s instrumented and what’s not. It also provides a goal post they can aim for. Currently, we encourage engineers to aim for 70% or higher, but it’s not enforced currently.\n\n## Next Steps\n\nThere are a few improvements we’re thinking of making. We want to decouple it from specific object types to simplify things even further. Engineers are less likely to write format-specific rules for the same piece of data. Our configuration has explicit support for data classes and Python dictionaries. However, I’ve thought of using it in other places where we’re using different data types but handling the same data. I want to transition it to where it thinks generically about primitive and complex object types (including nested ones), regardless of the format.\n\nWe also want to move beyond coverage. Coverage is a controversial metric in engineering. It may improve code quality but aiming too high can be pointless, arbitrary and force engineers to game the system. Coverage gives us a basic idea of what we’ll capture, but it falls short of the real picture. We want to know if the data will be captured in a particular workflow. It doesn’t matter that the data exists at a certain level. We’ll definitely want to extend this model so it’s more sophisticated.\n\nFinally, we want to capture function outputs too. I haven’t figured a great way to represent outputs given they’re unnamed in Python. I don’t want to couple the naming to the structure. Eventually we’ll figure that out.\n\n## Conclusion\n\nConfig-Accelerated Custom Instrumentation has been a game-changer for us. What started as a 10-year problem became something we could tackle in a reasonable timeframe. A configuration file, a metaclass, and a coverage report—and that's actually it!\n\nWe're still iterating and improving, but even in its current form, CACI has proven its worth. Our services are better instrumented, our incidents are easier to debug, and our engineers are spending less time manually adding telemetry and more time building features.\n\nIf you're staring down a similar mountain of under-instrumented services, I'd encourage you to think about what automation could do for you. The manual approach works, but it doesn't scale. And in a world where observability can make or break your ability to respond to incidents, waiting 10 years isn't "])</script><script>self.__next_f.push([1,"an option.\n\nThanks for stopping by!b:[\"$\",\"main\",null,{\"id\":\"main-content\",\"className\":\"min-h-screen bg-transparent\",\"children\":[[\"$\",\"$L17\",null,{\"publication\":{\"__typename\":\"Publication\",\"id\":\"625387b175dccb6f343206bb\",\"url\":\"https://jaywhy13.hashnode.dev\",\"canonicalURL\":\"https://jaywhy13.hashnode.dev\",\"urlPattern\":\"simple\",\"title\":\"Perspective Unspoken\",\"displayTitle\":null,\"hasBadges\":true,\"descriptionSEO\":\"Welcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for \",\"seo\":{\"title\":null,\"description\":null,\"__typename\":\"SEO\"},\"publicMembers\":{\"totalDocuments\":1,\"__typename\":\"PublicMembers\"},\"about\":{\"html\":\"\u003cp\u003eWelcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.\u003c/p\u003e\\n\",\"text\":\"Welcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.\\n\",\"__typename\":\"Content\"},\"features\":{\"proTeam\":{\"isEnabled\":false,\"__typename\":\"ProTeamFeature\"},\"newsletter\":{\"isEnabled\":true,\"__typename\":\"NewsletterFeature\"},\"viewCount\":{\"isEnabled\":false,\"__typename\":\"ViewCountFeature\"},\"readTime\":{\"isEnabled\":true,\"__typename\":\"ReadTimeFeature\"},\"textSelectionSharer\":{\"isEnabled\":true,\"__typename\":\"TextSelectionSharerFeature\"},\"gptBotCrawling\":{\"__typename\":\"GPTBotCrawlingFeature\",\"isEnabled\":false},\"headlessCMS\":{\"isEnabled\":false,\"__typename\":\"HeadlessCMSFeature\"},\"__typename\":\"PublicationFeatures\"},\"aeoSettings\":{\"isEnabled\":false,\"llmsTxtEnabled\":true,\"crawlers\":{\"gptBot\":false,\"claudeBot\":true,\"perplexityBot\":true,\"googleExtended\":true,\"ccBot\":true,\"__typename\":\"AeoCrawlerSettings\"},\"__typename\":\"PublicationAeoSettings\"},\"metaTags\":null,\"ogMetaData\":{\"image\":null,\"__typename\":\"OpenGraphMetaData\"},\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"name\":\"Jean-Mark Wright\",\"username\":\"jaywhy13\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\"},\"preferences\":{\"__typename\":\"PublicationPreferences\",\"logo\":null,\"darkMode\":{\"__typename\":\"DarkModePreferences\",\"logo\":null,\"enabled\":false},\"navbarItems\":[{\"__typename\":\"PublicationNavbarItem\",\"id\":\"66e232819d27b6c7f345ab6e\",\"label\":\"Calendar\",\"url\":\"https://calendar.google.com/calendar/u/0/appointments/AcZssZ3tWQd0YSITufTZaAitKeE6Ln2NBOouN5Qlop4=?gv=true\",\"type\":\"link\",\"series\":null,\"page\":null}],\"enabledPages\":{\"__typename\":\"EnabledPages\",\"badges\":true,\"newsletter\":false,\"members\":false},\"layout\":\"magazine\",\"disableFooterBranding\":false,\"isSubscriptionModalDisabled\":false},\"favicon\":null,\"headerColor\":\"#ff8484\",\"integrations\":{\"fbPixelID\":null,\"fathomSiteID\":null,\"fathomCustomDomainEnabled\":false,\"fathomCustomDomain\":null,\"hotjarSiteID\":null,\"matomoSiteID\":null,\"matomoURL\":null,\"gaTrackingID\":null,\"gTagManagerID\":null,\"plausibleAnalyticsEnabled\":false,\"wmPaymentPointer\":null,\"koalaPublicKey\":null,\"msClarityID\":null,\"openPanelClientId\":null,\"consentManagement\":{\"enabled\":false,\"version\":1,\"privacyPolicyURL\":null,\"__typename\":\"ConsentManagement\"},\"__typename\":\"PublicationIntegrations\"},\"imprintV2\":null,\"postsCount\":{\"totalDocuments\":19,\"__typename\":\"PostConnection\"},\"isTeam\":true,\"links\":{\"twitter\":\"https://twitter.com/kramnaej\",\"instagram\":null,\"github\":\"https://github.com/jaywhy13\",\"website\":null,\"hashnode\":\"https://hashnode.com/@jaywhy13\",\"youtube\":null,\"dailydev\":null,\"linkedin\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"mastodon\":null,\"facebook\":null,\"bluesky\":null,\"__typename\":\"PublicationLinks\"},\"domainInfo\":{\"__typename\":\"DomainInfo\",\"hashnodeSubdomain\":\"jaywhy13\",\"domain\":null,\"wwwPrefixedDomain\":null},\"redirectionRules\":[{\"source\":\"/two-inportant-lessons-i-learnt-rolling-out-a-new-platform-feature\",\"destination\":\"/two-important-lessons-i-learned-rolling-out-a-new-platform-feature\",\"type\":301,\"__typename\":\"RedirectionRule\"},{\"source\":\"/would-sherlock-use-traces-or-metric"])</script><script>self.__next_f.push([1,"s-to-debug-your-application\",\"destination\":\"/3-reasons-traces-better-than-metrics-for-debugging-your-application\",\"type\":301,\"__typename\":\"RedirectionRule\"}],\"customRules\":[{\"type\":302,\"path\":\"/linkedin\",\"destination\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"__typename\":\"CustomRule\"}],\"totalRecommendedPublications\":0,\"allowContributorEdits\":false,\"widgets\":[],\"post\":{\"__typename\":\"Post\",\"id\":\"694a1633103f879a13892085\",\"cuid\":\"cmji2h6zf000102jx6k7x1e0n\",\"title\":\"Instrumenting at 10 years per second\",\"subtitle\":\"How we made custom instrumentation blazing fast, simple, and data-centric\",\"slug\":\"instrumenting-at-10-years-per-second\",\"brief\":\"Introduction\\nI've found a new love - custom instrumentation, and I can’t keep it to myself. We manually increased the custom instrumentation in one of our newer services and loved the impact. The instrumentation accelerated forensic investigations, r...\",\"featured\":false,\"publishedAt\":\"2025-12-23T04:10:27.099Z\",\"updatedAt\":null,\"url\":\"https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second\",\"readTimeInMinutes\":8,\"views\":276,\"reactionCount\":0,\"responseCount\":0,\"replyCount\":0,\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"name\":\"Jean-Mark Wright\",\"username\":\"jaywhy13\",\"deactivated\":false,\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"bio\":{\"html\":\"\u003cp\u003eJean-Mark is an articulate, customer-focused, visionary with extensive industry experience encompassing front and back end development in all phases of software development. He lives \\\"outside the box\\\" in pursuit of architectural elegance and relevant, tangible solutions. He is a strong believer that team empowerment fosters genius and is versed at working solo.\u003c/p\u003e\\n\",\"__typename\":\"Content\"},\"socialMediaLinks\":{\"website\":\"\",\"github\":\"https://github.com/jaywhy13\",\"twitter\":\"https://twitter.com/kramnaej\",\"facebook\":\"\",\"stackoverflow\":\"\",\"linkedin\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"__typename\":\"SocialMediaLinks\"}},\"coAuthors\":[],\"seo\":{\"title\":\"Accelerating the pace of custom instrumentation\",\"description\":\"Struggling to instrument dozens of microservices? Discover our config-driven approach that accelerates custom instrumentation from years to weeks.\",\"__typename\":\"SEO\"},\"coverImage\":{\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg\",\"isPortrait\":false,\"attribution\":\"https://unsplash.com/@laurentmedia\",\"isAttributionHidden\":false,\"photographer\":\"toine G\",\"__typename\":\"CoverImage\"},\"content\":{\"html\":\"$18\",\"markdown\":\"$19\",\"__typename\":\"Content\"},\"preferences\":{\"pinnedToBlog\":false,\"disableComments\":false,\"stickCoverToBottom\":false,\"isDelisted\":false,\"__typename\":\"PostPreferences\"},\"series\":null,\"tags\":[{\"id\":\"5f1df3b4f4016901885ccc96\",\"slug\":\"observability\",\"name\":\"observability\",\"__typename\":\"Tag\"},{\"id\":\"5fc2287b55b779631e1acc29\",\"slug\":\"instrumentation\",\"name\":\"instrumentation\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b94d67\",\"slug\":\"python\",\"name\":\"Python\",\"__typename\":\"Tag\"},{\"id\":\"56744723958ef13879b95342\",\"slug\":\"python3\",\"name\":\"Python 3\",\"__typename\":\"Tag\"}],\"ogMetaData\":{\"image\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg\",\"__typename\":\"OpenGraphMetaData\"},\"canonicalUrl\":null,\"hasLatexInPost\":false,\"faq\":[],\"features\":{\"tableOfContents\":{\"isEnabled\":true,\"items\":[{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-0\",\"level\":2,\"slug\":\"introduction\",\"title\":\"Introduction\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-1\",\"level\":2,\"slug\":\"solution-goals\",\"title\":\"Solution Goals\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-2\",\"level\":2,\"slug\":\"how-it-works\",\"title\":\"How it works\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-3\",\"level\":3,\"slug\":\"the-configuration\",\"title\":\"The Configuration\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-4\",\"level\":3,\"slug\":\"the-hook\",\"title\":\"The Hook\",\"parentId\":null},{\"__typename\":\"Tabl"])</script><script>self.__next_f.push([1,"eOfContentsItem\",\"id\":\"toc-5\",\"level\":3,\"slug\":\"coverage-report\",\"title\":\"Coverage Report\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-6\",\"level\":2,\"slug\":\"next-steps\",\"title\":\"Next Steps\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-7\",\"level\":2,\"slug\":\"conclusion\",\"title\":\"Conclusion\",\"parentId\":null}],\"__typename\":\"TableOfContents\"},\"__typename\":\"PostFeatures\"}},\"posts\":{\"edges\":[{\"node\":{\"__typename\":\"Post\",\"id\":\"694a1633103f879a13892085\",\"title\":\"Instrumenting at 10 years per second\",\"slug\":\"instrumenting-at-10-years-per-second\",\"publishedAt\":\"2025-12-23T04:10:27.099Z\",\"cuid\":\"cmji2h6zf000102jx6k7x1e0n\",\"url\":\"https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second\",\"subtitle\":\"How we made custom instrumentation blazing fast, simple, and data-centric\",\"brief\":\"Introduction\\nI've found a new love - custom instrumentation, and I can’t keep it to myself. We manually increased the custom instrumentation in one of our newer services and loved the impact. The instrumentation accelerated forensic investigations, r...\",\"readTimeInMinutes\":8,\"views\":276,\"tags\":[{\"id\":\"5f1df3b4f4016901885ccc96\",\"slug\":\"observability\",\"name\":\"observability\",\"__typename\":\"Tag\"},{\"id\":\"5fc2287b55b779631e1acc29\",\"slug\":\"instrumentation\",\"name\":\"instrumentation\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b94d67\",\"slug\":\"python\",\"name\":\"Python\",\"__typename\":\"Tag\"},{\"id\":\"56744723958ef13879b95342\",\"slug\":\"python3\",\"name\":\"Python 3\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"67b2ab28c4ef92b38e5bdda0\",\"title\":\"On shipping smaller PRs\",\"slug\":\"on-shipping-smaller-prs\",\"publishedAt\":\"2025-02-17T03:21:12.558Z\",\"cuid\":\"cm78hnn8u000409l51hurfg9f\",\"url\":\"https://jaywhy13.hashnode.dev/on-shipping-smaller-prs\",\"subtitle\":\"Concrete examples for how to ship smaller PRs\",\"brief\":\"Introduction\\nIn my last I talked about the importance of shipping smaller PRs. In this article I’ll give some practical examples for how PRs can be split into smaller chunks. The examples aren’t organized into specific themes. Rather they’re arbitrar...\",\"readTimeInMinutes\":8,\"views\":83,\"tags\":[{\"id\":\"59c5f7f0ea46c0424704e8c1\",\"slug\":\"pull-requests\",\"name\":\"Pull Requests\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b949f9\",\"slug\":\"code-review\",\"name\":\"code review\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b94ad1\",\"slug\":\"software-development\",\"name\":\"software development\",\"__typename\":\"Tag\"},{\"id\":\"569d22c892921b8f79d35f68\",\"slug\":\"software-engineering\",\"name\":\"Software Engineering\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/o9oQaOGpLz0/upload/9e19fbfd378d65c448f7233b2a273b79.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"676b90a8cb76c2499dc67afb\",\"title\":\"Small PRs ship faster - QED\",\"slug\":\"small-prs-ship-faster-qed\",\"publishedAt\":\"2024-12-25T04:57:12.349Z\",\"cuid\":\"cm53fb3j1000309mb1rckcb01\",\"url\":\"https://jaywhy13.hashnode.dev/small-prs-ship-faster-qed\",\"subtitle\":\"My unrestrained passion for shipping smaller PRs constrained into a handful of benefits\",\"brief\":\"Introduction\\nWelcome to my very scientific paper where I propose that small PRs ship faster than big ones. I will construct my hypothesis with carefully crafted examples, concrete evidence and lots of words and sent"])</script><script>self.__next_f.push([1,"ences in general. I guarantee there...\",\"readTimeInMinutes\":5,\"views\":110,\"tags\":[{\"id\":\"56744721958ef13879b949f9\",\"slug\":\"code-review\",\"name\":\"code review\",\"__typename\":\"Tag\"},{\"id\":\"59c5f7f0ea46c0424704e8c1\",\"slug\":\"pull-requests\",\"name\":\"Pull Requests\",\"__typename\":\"Tag\"},{\"id\":\"5d9352eae235795f6eb6df65\",\"slug\":\"pull-request\",\"name\":\"pull request\",\"__typename\":\"Tag\"},{\"id\":\"56744723958ef13879b953b4\",\"slug\":\"review\",\"name\":\"review\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/JzMS3RLSZn8/upload/a18ee8f82d0d56644ab82f6c1bead8b4.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"671072daa4aac495b3ea5d5a\",\"title\":\"Order matters - making a compound index 50x faster\",\"slug\":\"order-matters-making-a-compound-index-50x-faster\",\"publishedAt\":\"2024-10-17T02:13:46.740Z\",\"cuid\":\"cm2co35h0000009ld0dg42agd\",\"url\":\"https://jaywhy13.hashnode.dev/order-matters-making-a-compound-index-50x-faster\",\"subtitle\":\"A story of how we made a compound index 50x faster by re-organizing the fields\",\"brief\":\"Introduction\\nToday I’ll talk about an endpoint that initially performed under 30ms, then crept up to ~500ms after a couple months. Our investigation revealed that a query, powered by a compound index was responsible for the elevated latency. We explo...\",\"readTimeInMinutes\":6,\"views\":1082,\"tags\":[{\"id\":\"56744722958ef13879b950eb\",\"slug\":\"databases\",\"name\":\"Databases\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b949b5\",\"slug\":\"postgresql\",\"name\":\"PostgreSQL\",\"__typename\":\"Tag\"},{\"id\":\"56744722958ef13879b94f0b\",\"slug\":\"postgres\",\"name\":\"postgres\",\"__typename\":\"Tag\"},{\"id\":\"64308297311bf43ae8efe880\",\"slug\":\"postgresql-performance\",\"name\":\"Postgresql-performance \",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b94821\",\"slug\":\"optimization\",\"name\":\"optimization\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1726107899370/04b1ac6d-7825-4a05-b5d0-0be78a2f49d9.png\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"66e239b9dcb487d824fc1b6a\",\"title\":\"Dear Editor: We need better Database Observability\",\"slug\":\"dear-editor-we-need-better-database-observability\",\"publishedAt\":\"2024-09-12T00:45:45.852Z\",\"cuid\":\"cm0ykj5do000708lj0q3laq8c\",\"url\":\"https://jaywhy13.hashnode.dev/dear-editor-we-need-better-database-observability\",\"subtitle\":\"A call for better database observability, or awareness to a better reality\",\"brief\":\"Introduction\\nDear Editor, hear my plea. I pen this piece in earnest search of enlightenment, or the beginnings of a well needed change; we need better database observability! Specifically, more granular, and useful database spans. Coarse, high-level ...\",\"readTimeInMinutes\":8,\"views\":745,\"tags\":[{\"id\":\"5f1df3b4f4016901885ccc96\",\"slug\":\"observability\",\"name\":\"observability\",\"__typename\":\"Tag\"},{\"id\":\"56744722958ef13879b950eb\",\"slug\":\"databases\",\"name\":\"Databases\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/NvFkYV2ngOk/upload/2f4cbc39f32355a334d9119af2dd086c.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"}],\"__typename\":\"PostConnection\"},\"redirectedPost\":null,\"staticPage\":null},\""])</script><script>self.__next_f.push([1,"posts\":[\"$b:props:children:0:props:publication:post\"],\"host\":\"jaywhy13.hashnode.dev\"}],\"$L1a\",\"$L1b\",null,\"$L1c\",\"$L1d\",null,\"$L1e\"]}]\n1f:I[62031,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"ReadingModeToggle\"]\n20:I[81356,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"Image\"]\n21:I[52619,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"8974\",\"static/chunks/app/page-cc18ae1a3a7a5ed8.js\"],\"\"]\n22:I[33884,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"Avatar\"]\n23:I[33884,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"AvatarImage\"]\n24:I[33884,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"AvatarFallback\"]\n27:I[51285,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"CommentsSection\"]\n28:I[20730,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"s"])</script><script>self.__next_f.push([1,"tatic/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"default\"]\n1a:[\"$\",\"section\",null,{\"className\":\"py-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"px-4 sm:px-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]\",\"children\":[\"$\",\"div\",null,{\"className\":\"grid gap-5 max-w-full\",\"children\":[\"$\",\"article\",null,{\"className\":\"w-full max-w-full rounded-xl border bg-card text-card-foreground\",\"children\":[[\"$\",\"div\",null,{\"className\":\"px-5 pt-12 pb-8 sm:px-8\",\"children\":[\"$\",\"header\",null,{\"className\":\"space-y-5\",\"children\":[\"$\",\"div\",null,{\"className\":\"space-y-4 text-center max-w-4xl mx-auto\",\"children\":[[\"$\",\"h1\",null,{\"className\":\"text-2xl sm:text-3xl md:text-4xl lg:text-5xl font-bold leading-tight tracking-tight text-balance text-foreground\",\"children\":\"Instrumenting at 10 years per second\"}],[\"$\",\"p\",null,{\"className\":\"text-lg sm:text-xl leading-normal font-normal text-muted-foreground text-balance\",\"children\":\"How we made custom instrumentation blazing fast, simple, and data-centric\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center justify-center gap-3 text-sm text-muted-foreground\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-2\",\"children\":[[\"$\",\"span\",null,{\"children\":\"Published\"}],[\"$\",\"time\",null,{\"dateTime\":\"2025-12-23T04:10:27.099Z\",\"children\":\"December 23, 2025\"}]]}],[[\"$\",\"span\",null,{\"className\":\"text-muted-foreground\",\"children\":\"•\"}],[\"$\",\"span\",null,{\"children\":[8,\" min read\"]}]],[\"$\",\"span\",null,{\"className\":\"text-muted-foreground\",\"children\":\"•\"}],[\"$\",\"$L1f\",null,{\"slug\":\"instrumenting-at-10-years-per-second\"}]]}]]}]}]}],[\"$\",\"div\",null,{\"className\":\"w-full\",\"children\":[\"$\",\"$L20\",null,{\"src\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg\",\"alt\":\"Instrumenting at 10 years per second\",\"width\":1200,\"height\":750,\"className\":\"w-full h-auto\",\"priority\":true}]}],[\"$\",\"div\",null,{\"className\":\"px-5 py-12 sm:px-8 space-y-8\",\"children\":[[\"$\",\"div\",null,{\"className\":\"max-w-2xl mx-auto\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center gap-5\",\"children\":[[\"$\",\"div\",\"6253878075dccb6f343206b7\",{\"className\":\"flex items-center gap-5\",\"children\":[false,[\"$\",\"div\",null,{\"className\":\"flex gap-3 items-start\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/@jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener\",\"children\":[\"$\",\"$L22\",null,{\"className\":\"h-10 w-10 border border-border transition-opacity hover:opacity-80\",\"children\":[[\"$\",\"$L23\",null,{\"src\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg?auto=compress,format\u0026format=webp\",\"alt\":\"Jean-Mark Wright\"}],[\"$\",\"$L24\",null,{\"className\":\"bg-muted text-muted-foreground font-semibold text-lg\",\"children\":\"J\"}]]}]}],[\"$\",\"div\",null,{\"className\":\"min-w-0\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-2\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/@jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener\",\"className\":\"font-medium text-foreground hover:text-primary transition-colors\",\"children\":\"Jean-Mark Wright\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center gap-1.5 text-sm\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"https://twitter.com/kramnaej\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground hover:text-foreground transition-colors\",\"aria-label\":\"X (Twitter)\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-x-twitter\",\"aria-hidden\":true}]}],[\"$\",\"$L21\",null,{\"href\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\""])</script><script>self.__next_f.push([1,":\"text-muted-foreground hover:text-foreground transition-colors\",\"aria-label\":\"LinkedIn\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-linkedin\",\"aria-hidden\":true}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"text-sm text-muted-foreground line-clamp-2\",\"dangerouslySetInnerHTML\":{\"__html\":\"\u003cp\u003eJean-Mark is an articulate, customer-focused, visionary with extensive industry experience encompassing front and back end development in all phases of software development. He lives \\\"outside the box\\\" in pursuit of architectural elegance and relevant, tangible solutions. He is a strong believer that team empowerment fosters genius and is versed at working solo.\u003c/p\u003e\\n\"}}]]}]]}]]}]]}]}],null,\"$L25\",false,\"$L26\"]}]]}]}]}]}]}]\n1b:[\"$\",\"section\",null,{\"className\":\"pb-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"px-4 sm:px-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]\",\"children\":[\"$\",\"div\",null,{\"className\":\"max-w-4xl w-full mx-auto\",\"children\":[\"$\",\"div\",null,{\"className\":\"rounded-xl border bg-card text-card-foreground p-5 sm:p-8\",\"children\":[\"$\",\"$L27\",null,{\"host\":\"jaywhy13.hashnode.dev\",\"slug\":\"instrumenting-at-10-years-per-second\",\"postUrl\":\"https://hashnode.com/posts/instrumenting-at-10-years-per-second/694a1633103f879a13892085\",\"initialEdges\":[],\"initialPageInfo\":{\"hasNextPage\":false,\"endCursor\":\"10\",\"totalDocuments\":0,\"__typename\":\"PageInfo\"},\"totalComments\":0}]}]}]}]}]}]\n1c:[\"$\",\"section\",null,{\"className\":\"pb-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"px-4 sm:px-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]\",\"children\":[\"$\",\"div\",null,{\"className\":\"max-w-4xl w-full mx-auto\",\"children\":[\"$\",\"section\",null,{\"className\":\"space-y-6\",\"children\":[[\"$\",\"h2\",null,{\"className\":\"text-2xl font-bold tracking-tight text-balance\",\"children\":\"More from this blog\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-col gap-6\",\"children\":[[\"$\",\"$L28\",\"67b2ab28c4ef92b38e5bdda0\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:1:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"676b90a8cb76c2499dc67afb\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:2:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"671072daa4aac495b3ea5d5a\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:3:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"66e239b9dcb487d824fc1b6a\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:4:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"div\",\"author-card\",{\"ref\":\"$undefined\",\"className\":\"rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 p-5 flex flex-col\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex flex-col items-center text-center gap-5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-col items-center gap-3\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex h-16 w-16 shrink-0 items-center justify-center rounded-full bg-muted\",\"children\":[\"$\",\"i\",null,{\"className\":\"text-2xl fa-brands fa-hashnode text-foreground\",\"aria-hidden\":true}]}],[\"$\",\"div\",null,{\"className\":\"space-y-1\",\"children\":[[\"$\",\"p\",null,{\"className\":\"text-base font-semibold text-foreground leading-tight\",\"children\":\"Perspective Unspoken\"}],[\"$\",\"p\",null,{\"className\":\"text-sm text-muted-foreground\",\"children\":[19,\" posts\"]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"prose prose-sm dark:prose-invert max-w-xs mx-auto text-muted-foreground\",\"dangerouslySetInnerHTML\":{\"__html\":\"\u003cp\u003eWelcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.\u003c/p\u003e\\n\"}}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center justify-center gap-3\",\"children\":[[],[[\"$\",\"a\",\"Twitter\",{\"href\":\"https://twitter.com/kramnaej\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"Twitter\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-x-twit"])</script><script>self.__next_f.push([1,"ter\",\"aria-hidden\":true}]}],[\"$\",\"a\",\"GitHub\",{\"href\":\"https://github.com/jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"GitHub\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-github\",\"aria-hidden\":true}]}],[\"$\",\"a\",\"LinkedIn\",{\"href\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"LinkedIn\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-linkedin\",\"aria-hidden\":true}]}],[\"$\",\"a\",\"Hashnode\",{\"href\":\"https://hashnode.com/@jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"Hashnode\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-hashnode\",\"aria-hidden\":true}]}]]]}]]}]}]]}]]}]}]}]}]}]\n29:T48b,{\"@context\":\"https://schema.org\",\"@type\":\"Article\",\"url\":\"https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second\",\"mainEntityOfPage\":\"https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second\",\"headline\":\"Instrumenting at 10 years per second\",\"description\":\"Introduction\\nI've found a new love - custom instrumentation, and I can’t keep it to myself. We manually increased the custom instrumentation in one of our newer services and loved the impact. The instrumentation accelerated forensic investigations, r...\",\"author\":{\"@type\":\"Person\",\"name\":\"Jean-Mark Wright\",\"url\":\"https://hashnode.com/@jaywhy13\",\"image\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"sameAs\":\"https://x.com/kramnaej\"},\"publisher\":{\"@type\":\"Organization\",\"name\":\"Perspective Unspoken\",\"url\":\"https://jaywhy13.hashnode.dev\"},\"datePublished\":\"2025-12-23T04:10:27.099Z\",\"dateModified\":\"2025-12-23T04:10:27.099Z\",\"image\":{\"@type\":\"ImageObject\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg\"},\"keywords\":\"observability, instrumentation, Python, Python 3\"}1d:[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"$29\"}}]\n2a:I[78655,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"TableOfContents\"]\n2b:T31b3,\u003ch2 id=\"heading-introduction\"\u003eIntroduction\u003c/h2\u003e\n\u003cp\u003eI've found a new love - custom instrumentation, and I can’t keep it to myself. We manually increased the custom instrumentation in one of our newer services and loved the impact. The instrumentation accelerated forensic investigations, root cause analysis and system comprehension. I’m hopelessly in love because I experienced worse. I’ve felt the pain of data insufficiency. We’ve had too many incidents where our traces could barely help us identify the impacted customer segment, much less tell us how many customers were impacted. Custom instrumentation has demonstrated its value, and we wanted to share it with the rest of the microservices in our fleet.\u003c/p\u003e\n\u003cp\u003eThere was only one problem. Although we had one well-instrumented microservice, we still had over 40 other microservices that needed to be instrumented. Adding high-quality custom instrumentation takes time. Assuming it’d take about 3 months to instrument each of those services, it’d take ~10 years to get to the quality we needed. And which engineering leadership team doesn’t like to hear a feature which takes only 10 years to build? So… yah… we sought to fix that problem.\u003c/p\u003e\n\u003cp\u003eThe rest of the talk covers a new approach we cr"])</script><script>self.__next_f.push([1,"eated called Config-Accelerated Custom Instrumentation, which we’re using to accelerate the time taken to add high-quality custom instrumentation to our services.\u003c/p\u003e\n\u003ch2 id=\"heading-solution-goals\"\u003eSolution Goals\u003c/h2\u003e\n\u003cp\u003eWe had a few goals in mind for building an automated solution.\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eSpeed\u003c/strong\u003e - We want instrumentation creation to be really fast too. We had ~40+ services in our fleet that needed this tooling, and 10 years wasn’t going to cut it.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eSimplicity\u003c/strong\u003e - we wanted a solution that enabled engineers to add instrumentation easily. Relative to the manual effort, it should be a piece of cake to get quality instrumentation captured. We want engineers to enjoy adding telemetry.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eData-centric\u003c/strong\u003e - we want our engineers to think simply about the data they want to capture. We’d do the heavy lifting and capture the data for them. So, we want them to think “these are the data points I want to capture”. We’d provide the necessary automation and logic to observe the application stack and collect the data.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eSmall footprint\u003c/strong\u003e - we also wanted a super unobtrusive implementation. We don’t want our tooling littering the codebase. A large footprint puts a real damper on engineer productivity and any evolutions we want to make.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eVerifiability\u003c/strong\u003e - we want engineers to verify that the instrumentation works as expected. Our desire for simplicity would likely remove some visibility, so we want to restore that visibility with verifiable mechanisms. We needed to answer questions like “will this datapoint be captured in my telemetry?\" or “which areas of my app are under-instrumented?”\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003cstrong\u003eSecure\u003c/strong\u003e - we don't want to risk any PII leakages. If we’re automating custom instrumentation, we’ll likely touch some PII and we’ll need to be careful nothing’s unintentionally leaked.\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003ch2 id=\"heading-how-it-works\"\u003eHow it works\u003c/h2\u003e\n\u003cp\u003eOur Config-Accelerated Custom Instrumentation (CACI) approach consisted of three distinct pieces. There’s the configuration that tells us what to instrument, a hook (metaclass) that captures the telemetry from classes and a report that provides feedback on what’ll be captured. Altogether, this simple set of tools helps us accelerate the pace of custom instrumentation dramatically!\u003c/p\u003e\n\u003ch3 id=\"heading-the-configuration\"\u003eThe Configuration\u003c/h3\u003e\n\u003cp\u003eLet’s talk about the Configuration first. The configuration provides guidelines for what our tooling should capture for engineers. This is where engineers tell us what they want to instrument. The config captures allowed field names, field types, data classes and dictionaries. It functions as a purely opt-in configuration. Data has to be explicitly included; nothing is captured by default. Here’s what a sample configuration looks like:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003ecapture_configuration = TelemetryCaptureConfiguration(\n allowed_field_names=[\n \u003cspan class=\"hljs-string\"\u003e\"account_id\"\u003c/span\u003e,\n \u003cspan class=\"hljs-string\"\u003e\"business_id\"\u003c/span\u003e,\n ],\n disallowed_field_names=[],\n allowed_field_types=[UUID, datetime, bool, Enum],\n disallowed_field_types=[],\n allowed_dataclasses=[\n \u003cspan class=\"hljs-comment\"\u003e# Common\u003c/span\u003e\n AllowedDataclass(\n dataclass_name=\u003cspan class=\"hljs-string\"\u003e\"Business\"\u003c/span\u003e,\n allowed_field_names=[\n \u003cspan class=\"hljs-string\"\u003e\"business_id\"\u003c/span\u003e,\n ...\n ],\n ),\n ],\n)\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThe configuration places the focus squarely on the application’s data. We want the configuration to be completely decoupled from the application’s architecture. This avoids fragility and is change-resistant. We don’t want engineers having to update their configuration every time they add a new function, or change an existing one. The configuration is deliberately simple, easy to maintain and extend. Using this, engineers can tell us what "])</script><script>self.__next_f.push([1,"application data is important for us to capture. Once this configuration exists, we can collect the data with our hooks!\u003c/p\u003e\n\u003ch3 id=\"heading-the-hook\"\u003eThe Hook\u003c/h3\u003e\n\u003cp\u003eOnce we’ve got the configuration telling us what needs to be captured, now we need a mechanism for collecting the data exchanged between components in the application stack. We want to capture information exchanged by components. We started by capturing the function inputs for the component. So, effectively, every time some component calls another component, we want to inspect the function input to see if any of the variables are specified in the configuration. To enable this, we created a metaclass that wraps each public method, captures relevant inputs then passes through outputs and errors. We only want to do this for public methods to keep things simple and easy to understand. We opted to use a metaclass as our hook. Using a metaclass, we can hook into the class creation and refine the methods dynamically, inserting our logic.\u003c/p\u003e\n\u003cp\u003eThe basic logic looks like this…\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eComponentMetaclass\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003etype\u003c/span\u003e):\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003e__new__\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e...\u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Loop over all methods\u003c/span\u003e\n \u003cspan class=\"hljs-keyword\"\u003efor\u003c/span\u003e method_name, method \u003cspan class=\"hljs-keyword\"\u003ein\u003c/span\u003e methods:\n \u003cspan class=\"hljs-comment\"\u003e# Wrap the method with our logic\u003c/span\u003e\n new_method = wrap_method_with_our_logic(method)\n \u003cspan class=\"hljs-comment\"\u003e# Replace the method definition\u003c/span\u003e\n attrs[method_name] = new_method\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThen, using it is really easy. We just specify the metaclass on any class to use it.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionService\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003emetaclass=ComponentMetaclass\u003c/span\u003e):\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003eself, business_id: BusinessId, user_id: UserId\u003c/span\u003e):\u003c/span\u003e\n ...\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eAnd that’s it! Just by specifying that a component class wants to use our metaclass, our Observability package gets to work! It starts intercepting all calls. For instance, when a call is made to \u003ccode\u003eSubscriptionService().create_subscription(….)\u003c/code\u003e it calls our logic and checks if the \u003ccode\u003ebusiness_id\u003c/code\u003e and \u003ccode\u003euser_id\u003c/code\u003e are in the configuration (\u003ccode\u003eTelemetryCaptureConfiguration\u003c/code\u003e). If it is, we capture those inputs for submission later.\u003c/p\u003e\n\u003ch3 id=\"heading-coverage-report\"\u003eCoverage Report\u003c/h3\u003e\n\u003cp\u003eAn important aspect of our design was to provide an explicit feedback loop engineers could use. These tools enable engineers to cover more ground faster, however, it also removes considerable visibility for them. Previously, an engineer could look at a method and see what’s instrumented. In the example below you can see where engineers would manually stuff data into a \u003ccode\u003ectx\u003c/code\u003e. It was a tremendous feedback loop. Engineers can see the code and touch it. They can write test assertions to ensure it works as expected. By adding a configuration and these hooks, we’re granting power but removing visibility.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionService\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003eself, ctx, business: BusinessId, user_id: UserId\u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Capture some telemetry\u003c/span\u003e\n ctx[\u003cspan class=\"hljs-string\"\u003e\"business_id\"\u003c/span\u003e] = business_id\n ctx"])</script><script>self.__next_f.push([1,"[\u003cspan class=\"hljs-string\"\u003e\"user_id\"\u003c/span\u003e] = user_id\n\n \u003cspan class=\"hljs-comment\"\u003e# do actual logic ...\u003c/span\u003e\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eTo replace this visibility, we provide engineers with a Coverage Report. It’s a report that generates a method-by-method commentary on what will or won’t get captured by our tooling. It also provides a percentage of how many variables are captured.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-plaintext\"\u003e- SubscriptionService\n - create_subscription(self, business_id: BusinessId, user_id: UserId)\n - Inputs\n - business_id:BusinessId - ✅\n - user_id:UserId - 🚫\n\n-----------------------------------------------------------------\nParameters captured: 1/2 (50%)\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThis restores visibility for engineers aiming to understand what’s instrumented and what’s not. It also provides a goal post they can aim for. Currently, we encourage engineers to aim for 70% or higher, but it’s not enforced currently.\u003c/p\u003e\n\u003ch2 id=\"heading-next-steps\"\u003eNext Steps\u003c/h2\u003e\n\u003cp\u003eThere are a few improvements we’re thinking of making. We want to decouple it from specific object types to simplify things even further. Engineers are less likely to write format-specific rules for the same piece of data. Our configuration has explicit support for data classes and Python dictionaries. However, I’ve thought of using it in other places where we’re using different data types but handling the same data. I want to transition it to where it thinks generically about primitive and complex object types (including nested ones), regardless of the format.\u003c/p\u003e\n\u003cp\u003eWe also want to move beyond coverage. Coverage is a controversial metric in engineering. It may improve code quality but aiming too high can be pointless, arbitrary and force engineers to game the system. Coverage gives us a basic idea of what we’ll capture, but it falls short of the real picture. We want to know if the data will be captured in a particular workflow. It doesn’t matter that the data exists at a certain level. We’ll definitely want to extend this model so it’s more sophisticated.\u003c/p\u003e\n\u003cp\u003eFinally, we want to capture function outputs too. I haven’t figured a great way to represent outputs given they’re unnamed in Python. I don’t want to couple the naming to the structure. Eventually we’ll figure that out.\u003c/p\u003e\n\u003ch2 id=\"heading-conclusion\"\u003eConclusion\u003c/h2\u003e\n\u003cp\u003eConfig-Accelerated Custom Instrumentation has been a game-changer for us. What started as a 10-year problem became something we could tackle in a reasonable timeframe. A configuration file, a metaclass, and a coverage report—and that's actually it!\u003c/p\u003e\n\u003cp\u003eWe're still iterating and improving, but even in its current form, CACI has proven its worth. Our services are better instrumented, our incidents are easier to debug, and our engineers are spending less time manually adding telemetry and more time building features.\u003c/p\u003e\n\u003cp\u003eIf you're staring down a similar mountain of under-instrumented services, I'd encourage you to think about what automation could do for you. The manual approach works, but it doesn't scale. And in a world where observability can make or break your ability to respond to incidents, waiting 10 years isn't an option.\u003c/p\u003e\n\u003cp\u003eThanks for stopping by!\u003c/p\u003e\n25:[\"$\",\"div\",null,{\"className\":\"relative\",\"children\":[[\"$\",\"aside\",null,{\"className\":\"hidden xl:block absolute top-0 left-0 h-full\",\"children\":[\"$\",\"nav\",null,{\"className\":\"sticky top-22 w-44\",\"children\":[\"$\",\"$L2a\",null,{\"headings\":[{\"id\":\"heading-introduction\",\"text\":\"Introduction\",\"level\":2},{\"id\":\"heading-solution-goals\",\"text\":\"Solution Goals\",\"level\":2},{\"id\":\"heading-how-it-works\",\"text\":\"How it works\",\"level\":2},{\"id\":\"heading-the-configuration\",\"text\":\"The Configuration\",\"level\":3},{\"id\":\"heading-the-hook\",\"text\":\"The Hook\",\"level\":3},{\"id\":\"heading-coverage-report\",\"text\":\"Coverage Report\",\"level\":3},{\"id\":\"heading-next-steps\",\"text\":\"Next Steps\",\"level\":2},{\"id\":\"heading-conclusion\",\"text\":\"Conclusion\",\"level\":2}]}]}]}],[\"$\",\"div\",null,{\"className\":\"max-w-2xl mx-auto min-w-0 contain-[inline-size]\",\"children\":[\"$"])</script><script>self.__next_f.push([1,"\",\"div\",null,{\"className\":\"prose dark:prose-invert prose-pre:bg-foreground prose-pre:text-background prose-pre:border prose-pre:border-border prose-pre:rounded prose-pre:px-5 prose-pre:py-4 dark:prose-pre:bg-muted dark:prose-pre:text-foreground dark:prose-pre:border-border prose-pre:overflow-x-auto max-w-none text-base sm:text-lg [\u0026\u003ediv\u003ep:first-child]:mt-0 [\u0026\u003ediv\u003ep:first-child]:pt-0 min-w-0 wrap-break-word [\u0026_a]:break-all **:max-w-full\",\"children\":[[],[\"$\",\"div\",null,{\"suppressHydrationWarning\":true,\"dangerouslySetInnerHTML\":{\"__html\":\"$2b\"}}],[],\"$L2c\",\"$L2d\",\"$L2e\",\"$L2f\",\"$L30\"]}]}]]}]\n26:[\"$\",\"div\",null,{\"className\":\"max-w-2xl mx-auto border-t border-border pt-8 space-y-5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center gap-2\",\"children\":[[\"$\",\"$L21\",\"5f1df3b4f4016901885ccc96\",{\"href\":\"/tag/observability\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"observability\"]}],[\"$\",\"$L21\",\"5fc2287b55b779631e1acc29\",{\"href\":\"/tag/instrumentation\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"instrumentation\"]}],[\"$\",\"$L21\",\"56744721958ef13879b94d67\",{\"href\":\"/tag/python\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"python\"]}],[\"$\",\"$L21\",\"56744723958ef13879b95342\",{\"href\":\"/tag/python3\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"python3\"]}]]}],false]}]\n31:I[32478,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"CodeHighlighter\"]\n32:I[7714,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"EmbedHandler\"]\n33:I[54960,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"ImageZoom\"]\n34:I[6671,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunk"])</script><script>self.__next_f.push([1,"s/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"MathJaxHandler\"]\n35:I[91337,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"MermaidHandler\"]\n2c:[\"$\",\"$L31\",null,{}]\n2d:[\"$\",\"$L32\",null,{}]\n2e:[\"$\",\"$L33\",null,{}]\n2f:[\"$\",\"$L34\",null,{\"hasLatex\":false}]\n30:[\"$\",\"$L35\",null,{}]\n36:Tc98,M268.379 39q-6.322 0-10.35-4.108-4.002-4.134-4.002-10.483 0-6.296 4.002-10.43 4-4.135 10.163-4.135 6.028 0 9.817 3.734 3.814 3.735 3.814 9.497v2.8h-24.248v-4.054h17.633q-.187-2.747-2.054-4.508-1.868-1.76-4.962-1.76-3.28-.001-5.388 2.374-2.108 2.373-2.108 6.375 0 4.188 2.348 6.695 2.347 2.481 5.895 2.481 1.68 0 2.934-.453a8.5 8.5 0 0 0 2.321-1.307q1.041-.854 2.268-2.614l4.694 2.96q-1.707 2.748-3.494 4.135a12.7 12.7 0 0 1-3.975 2.081q-2.187.72-5.308.72m-33.496 0q-5.575 0-9.283-4.028-3.681-4.029-3.681-10.484 0-6.43 3.761-10.51 3.76-4.108 9.47-4.108 2.8 0 5.175 1.2 2.4 1.174 3.628 3.148l.026-.08V0h7.016v38.2h-6.722v-4.722l.64.8-.694.133q-1.28 2.108-3.788 3.361Q237.924 39 234.883 39m1.627-6.002q3.308 0 5.495-2.428t2.188-6.135q0-3.762-2.214-6.109-2.214-2.374-5.469-2.374-3.28 0-5.522 2.347-2.214 2.348-2.214 6.163 0 3.76 2.214 6.162 2.214 2.374 5.522 2.374M205.811 39q-6.376 0-10.537-4.161-4.161-4.188-4.161-10.404 0-6.321 4.161-10.456 4.188-4.135 10.537-4.135 6.348 0 10.537 4.161 4.188 4.161 4.188 10.43 0 6.216-4.188 10.404Q212.186 39 205.811 39m0-6.162q3.307 0 5.575-2.374 2.267-2.374 2.267-6.055 0-3.707-2.294-6.056-2.267-2.347-5.548-2.347-3.308 0-5.575 2.374-2.268 2.348-2.268 6.029t2.241 6.055q2.268 2.374 5.602 2.374M162.586 38.2V10.644h6.802v5.015l-.8-.987h.906q1.308-2.187 3.762-3.494 2.48-1.335 5.228-1.334 4.615 0 7.256 2.8 2.667 2.776 2.667 7.603V38.2h-7.069V21.794q0-2.854-1.387-4.401t-4.134-1.547q-2.614 0-4.402 1.974-1.76 1.974-1.76 4.881V38.2zm-30.181 0V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.069V21.794q0-2.827-1.387-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.814 2.027-1.814 4.962v15.391zm-14.416.8q-2.882 0-5.015-.64-2.108-.614-3.761-1.84-1.654-1.228-2.668-2.695l-.987-1.467 4.695-3.174.773 1.067a9.1 9.1 0 0 0 1.788 1.814 8.2 8.2 0 0 0 2.32 1.2q1.308.4 3.068.4 2.161 0 3.441-.8 1.308-.8 1.307-2.294 0-1.36-1.147-2.108-1.12-.747-5.068-1.654-5.335-1.253-7.656-3.334-2.32-2.107-2.321-5.362 0-3.76 3.148-6.002 3.148-2.267 7.923-2.267 2.454 0 4.348.507t3.388 1.493q1.493.988 2.454 2.188l.987 1.2-4.482 3.255-.667-.774a7.4 7.4 0 0 0-1.627-1.387 6.4 6.4 0 0 0-2.054-.88 9.5 9.5 0 0 0-2.401-.294q-1.974 0-3.201.8-1.2.774-1.2 1.921 0 1.254 1.12 2.027 1.147.748 5.229 1.681 5.334 1.227 7.629 3.308 2.294 2.055 2.294 5.308 0 4.029-3.228 6.43Q123.217 39 117.989 39M87.61 39q-4.508 0-7.335-2.454-2.828-2.481-2.828-6.482 0-4.188 3.254-6.669t8.483-2.48q2.082 0 4.028.426 1.974.427 3.361 1.12v-1.6q0-2.588-1.813-4.082-1.788-1.493-4.615-1.493-1.628 0-2.801.373a8.7 8.7 0 0 0-2.188.987q-.987.614-1.84 1.414l-.854.8-4.081-3.788 1.307-1.173a18.7 18.7 0 0 1 2.934-2.108q1.601-.933 3.601-1.44t4.589-.507q5.868 0 9.229 2.88 3.361 2.882 3.361 8.084V38.2h-6.749v-5.922l1.654 2.187h-1.76q-1.308 2.055-3.655 3.308Q90.545 39 87.611 39m1.814-4.988q3.148 0 5.149-2.187 2-2.215 2-5.309v-.08q-1.12-.693-2.72-1.094a13 13 0 0 0-3.388-.426q-2.828 0-4.562 1.2-1.734 1.173-1.734 3.521 0 2.028 1.468 3.201t3.787 1.174M49 38.2V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.07V21.794q0-2.827-1.386-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.815 2.027-1.814 4.962v15.391z1e:[\"$\",\"footer\",null,{\"className\":\"px-4 pt-8 pb-20 sm:px-8 text-muted-foreground\",\"ari"])</script><script>self.__next_f.push([1,"a-label\":\"Footer\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-4xl mx-auto\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex flex-col items-center gap-4 text-center text-sm\",\"children\":[false,[\"$\",\"p\",null,{\"className\":\"font-medium\",\"children\":[[\"$\",\"time\",null,{\"dateTime\":\"2026\",\"children\":[[\"$\",\"span\",null,{\"className\":\"font-sans\",\"children\":\"©\"}],\" \",2026]}],\" \",\"Perspective Unspoken\"]}],[\"$\",\"nav\",null,{\"aria-label\":\"Footer navigation\",\"children\":[\"$\",\"ul\",null,{\"className\":\"flex flex-wrap justify-center gap-4 text-sm tracking-wide\",\"children\":[[\"$\",\"li\",\"/archive\",{\"children\":[\"$\",\"$L21\",null,{\"href\":\"/archive\",\"className\":\"transition hover:text-foreground\",\"target\":\"$undefined\",\"rel\":\"$undefined\",\"children\":\"Archive\"}]}],[\"$\",\"li\",\"https://hashnode.com/privacy-policy\",{\"children\":[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/privacy-policy\",\"className\":\"transition hover:text-foreground\",\"target\":\"_blank\",\"rel\":\"noopener\",\"children\":\"Privacy\"}]}],[\"$\",\"li\",\"https://hashnode.com/terms\",{\"children\":[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/terms\",\"className\":\"transition hover:text-foreground\",\"target\":\"_blank\",\"rel\":\"noopener\",\"children\":\"Terms\"}]}]]}]}],[\"$\",\"div\",null,{\"className\":\"pt-8 flex justify-center gap-6\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"/sitemap.xml\",\"target\":\"_blank\",\"rel\":\"noopener\",\"className\":\"flex items-center gap-2 text-sm transition hover:text-foreground\",\"title\":\"Sitemap\",\"children\":[[\"$\",\"i\",null,{\"className\":\"fa-regular fa-sitemap\"}],[\"$\",\"span\",null,{\"children\":\"Sitemap\"}]]}],[\"$\",\"$L21\",null,{\"href\":\"/rss.xml\",\"target\":\"_blank\",\"rel\":\"noopener\",\"className\":\"flex items-center gap-2 text-sm transition hover:text-foreground\",\"title\":\"RSS Feed\",\"children\":[[\"$\",\"i\",null,{\"className\":\"fa-regular fa-rss\"}],[\"$\",\"span\",null,{\"children\":\"RSS\"}]]}]]}],[\"$\",\"div\",null,{\"className\":\"pt-10 text-center\",\"children\":[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev\u0026utm_medium=referral\u0026utm_campaign=blog_footer_logo\u0026utm_content=logo\",\"target\":\"_blank\",\"rel\":\"noopener\",\"aria-label\":\"Visit Hashnode (opens in a new tab)\",\"children\":[\"$\",\"svg\",null,{\"fill\":\"none\",\"viewBox\":\"0 0 282 39\",\"aria-hidden\":\"true\",\"className\":\"h-6 w-auto min-w-[9.5rem] text-muted-foreground transition hover:text-foreground\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"currentColor\",\"d\":\"M2.68 13.032c-3.573 3.505-3.573 9.363 0 12.936L13.032 36.32c3.505 3.573 9.363 3.573 12.936 0L36.32 25.968c3.573-3.573 3.573-9.431 0-12.936L25.968 2.68c-3.573-3.573-9.431-3.573-12.936 0zm12.211 1.935c2.507-2.521 6.582-2.544 9.104-.038s2.544 6.582.038 9.104-6.582 2.544-9.104.038-2.544-6.582-.038-9.104\"}],[\"$\",\"path\",null,{\"fill\":\"currentColor\",\"d\":\"$36\"}]]}],\"className\":\"inline-flex items-center justify-center gap-2 whitespace-nowrap font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [\u0026_svg]:pointer-events-none [\u0026_svg]:size-4 [\u0026_svg]:shrink-0 hover:bg-accent hover:text-accent-foreground h-10 rounded-md text-xl px-5 py-4\",\"ref\":null}]}]]}]}]}]\n"])</script></body></html>