Files
nexus/sreweekly/articles/433/02-automating-telemetry-capture-in-python-using-bytecode.html
2026-09-12 17:23:01 +08:00

336 lines
243 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="image" href="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg"/><link rel="stylesheet" href="/_next/static/css/bc5c0eaecd39bc7d.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/7ffcc02a3c67389b.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/c20e76e7d11dec32.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/a6eb8c1f37be68d5.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-d716a75e4017380e.js"/><script src="/_next/static/chunks/4bd1b696-100b9d70ed4e49c1.js" async=""></script><script src="/_next/static/chunks/9da6db1e-8a4e8c23641c31bf.js" async=""></script><script src="/_next/static/chunks/1255-a242e0ce0372fddf.js" async=""></script><script src="/_next/static/chunks/main-app-bda8a52fe841b3aa.js" async=""></script><script src="/_next/static/chunks/4554-984b480a9db5f3fe.js" async=""></script><script src="/_next/static/chunks/7563-4dfa0994a9b0e37c.js" async=""></script><script src="/_next/static/chunks/9216-9272709ba40c1cfd.js" async=""></script><script src="/_next/static/chunks/3874-323555cd5c106ba4.js" async=""></script><script src="/_next/static/chunks/3091-bf1537ff6571a1bd.js" async=""></script><script src="/_next/static/chunks/app/layout-f61a55efa656d91f.js" async=""></script><script src="/_next/static/chunks/app/not-found-b861935655c5698b.js" async=""></script><script src="/_next/static/chunks/5001-fae0bebdd93f95e3.js" async=""></script><script src="/_next/static/chunks/2423-c8b1388240338d7d.js" async=""></script><script src="/_next/static/chunks/652-f40dcb480b4e356f.js" async=""></script><script src="/_next/static/chunks/730-8191af9e9424b148.js" async=""></script><script src="/_next/static/chunks/app/page-cc18ae1a3a7a5ed8.js" async=""></script><script src="/_next/static/chunks/0dbeb660-9e82e99813552576.js" async=""></script><script src="/_next/static/chunks/1545-7abf96433ff6553f.js" async=""></script><script src="/_next/static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js" async=""></script><meta name="next-size-adjust" content=""/><title>Automating Telemetry Capture in Python using Bytecode</title><meta name="description" content="Discover our journey of automating telemetry capture in Python using bytecode hooks, the challenges we faced, and the alternatives we&#x27;re considering"/><link rel="canonical" href="https://jaywhy13.hashnode.dev/automated-telemetry-capture-via-python-bytecode-modification"/><meta property="og:title" content="Automating Telemetry Capture in Python using Bytecode"/><meta property="og:description" content="Discover our journey of automating telemetry capture in Python using bytecode hooks, the challenges we faced, and the alternatives we&#x27;re considering"/><meta property="og:url" content="https://jaywhy13.hashnode.dev/automated-telemetry-capture-via-python-bytecode-modification"/><meta property="og:site_name" content="Perspective Unspoken"/><meta property="og:image" content="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg"/><meta property="og:image:width" content="1200"/><meta property="og:image:height" content="630"/><meta property="og:type" content="article"/><meta property="article:published_time" content="2024-06-25T01:18:44.170Z"/><meta property="article:author" content="https://jaywhy13.hashnode.dev/@jaywhy13"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:creator" content="https://twitter.com/kramnaej"/><meta name="twitter:title" content="Automating Telemetry Capture in Python using Bytecode"/><meta name="twitter:description" content="Discover our journey of automating telemetry capture in Python using bytecode hooks, the challenges we faced, and the alternatives we&#x27;re considering"/><meta name="twitter:image" content="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg"/><link rel="icon" href="/favicon.ico"/><script type="text/javascript">window.addEventListener('message',function(e){var d=e.data;if(!d||d.__hnWidget!==1||typeof d.h!=='number'||d.h<=0)return;var f=document.querySelectorAll('.hn-widget-frame');for(var i=0;i<f.length;i++){if(f[i].contentWindow===e.source){f[i].style.height=d.h+'px';break;}}});</script><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body class="__variable_ede2c9 __variable_558dcf bg-background dark:bg-background"><div hidden=""><!--$--><!--/$--></div><script>((a2, b2, c4, d3, e2, f2, g4, h3) => {
let i3 = document.documentElement, j2 = ["light", "dark"];
function k2(b3) {
var c5;
(Array.isArray(a2) ? a2 : [a2]).forEach((a3) => {
let c6 = a3 === "class", d4 = c6 && f2 ? e2.map((a4) => f2[a4] || a4) : e2;
c6 ? (i3.classList.remove(...d4), i3.classList.add(f2 && f2[b3] ? f2[b3] : b3)) : i3.setAttribute(a3, b3);
}), c5 = b3, h3 && j2.includes(c5) && (i3.style.colorScheme = c5);
}
if (d3) k2(d3);
else try {
let a3 = localStorage.getItem(b2) || c4, d4 = g4 && a3 === "system" ? window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light" : a3;
k2(d4);
} catch {
}
})("class","theme","system",null,["light","dark"],null,true,true)</script><main id="main-content" class="min-h-screen bg-transparent"><header class="sticky top-0 z-50 w-full py-4 bg-background backdrop-blur-sm transition-transform duration-300 translate-y-0"><a href="#main-content" class="sr-only focus:not-sr-only focus:absolute focus:left-4 focus:top-4 focus:z-50 focus:rounded-md focus:bg-background focus:px-4 focus:py-2 focus:text-sm focus:shadow">Skip to main content</a><div class="relative w-full mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem] px-4 sm:px-12 md:px-18 lg:px-16 xl:px-10"><div class="flex items-center justify-between gap-3 xl:hidden"><div class="flex items-center gap-3 min-w-0 flex-1"><a href="https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev&amp;utm_medium=referral&amp;utm_campaign=blog_header_logo&amp;utm_content=logo" class="inline-flex items-center justify-center text-lg text-muted-foreground hover:text-foreground transition border rounded-md h-9 w-9 shrink-0 border-border hover:bg-muted"><i class="text-xl fa-brands fa-hashnode" aria-hidden="true"></i><span class="sr-only">Hashnode</span></a><a title="Perspective Unspoken" class="text-lg font-semibold text-foreground truncate min-w-0" href="/">Perspective Unspoken</a></div><div class="flex items-center gap-2 shrink-0"><button class="gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent py-2 h-9 w-9 px-0 flex items-center justify-center text-muted-foreground hover:text-foreground bg-transparent border-border shadow-none" aria-label="Open search (press Control or Command and K)"><i class="fa-jelly fa-search text-sm" aria-hidden="true"></i><span class="sr-only">Open search (press Control or Command and K)</span></button><button class="inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent w-9 px-3 h-9 text-muted-foreground hover:text-foreground shadow-none" type="button" id="radix-_R_2j4anpfalb_" aria-haspopup="menu" aria-expanded="false" data-state="closed" data-slot="dropdown-menu-trigger"><i class="fa-jelly fa-sun text-sm scale-100 rotate-0 transition-all dark:scale-0 dark:-rotate-90" aria-hidden="true"></i><i class="fa-jelly fa-moon absolute text-sm scale-0 rotate-90 transition-all dark:scale-100 dark:rotate-0" aria-hidden="true"></i><span class="sr-only">Toggle theme</span></button><button class="inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 border border-input bg-background hover:bg-accent hover:text-accent-foreground py-2 h-9 w-10 px-0 shadow-none" type="button" aria-haspopup="dialog" aria-expanded="false" aria-controls="radix-_R_3j4anpfalb_" data-state="closed" data-slot="sheet-trigger"><i class="fa-jelly fa-bars" aria-hidden="true"></i><span class="sr-only">Open menu</span></button></div></div><div class="hidden xl:grid gap-4 items-center xl:grid-cols-[1fr_auto_1fr] xl:gap-6 max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="flex items-center w-full gap-3 min-w-0"><a href="https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev&amp;utm_medium=referral&amp;utm_campaign=blog_header_logo&amp;utm_content=logo" class="inline-flex items-center justify-center text-lg text-muted-foreground hover:text-foreground transition border rounded-md h-9 w-9 border-border hover:bg-muted"><i class="text-xl fa-brands fa-hashnode" aria-hidden="true"></i><span class="sr-only">Hashnode</span></a><span class="h-6 border-l border-border"></span><p class="font-semibold text-foreground min-w-0 truncate text-lg sm:text-xl"><a title="Perspective Unspoken" href="/">Perspective Unspoken</a></p></div><nav aria-label="Main navigation" class="flex justify-center w-full"><nav aria-label="Main" data-orientation="horizontal" dir="ltr" data-slot="navigation-menu" data-viewport="false" class="group/navigation-menu relative flex flex-1 items-center max-w-max justify-center"><div style="position:relative"><ul data-orientation="horizontal" data-slot="navigation-menu-list" class="group flex flex-1 list-none items-center flex-nowrap justify-center gap-1 text-muted-foreground text-base" dir="ltr"><li data-slot="navigation-menu-item" class="relative"><a target="_blank" rel="noopener" data-slot="navigation-menu-link" class="data-[active=true]:focus:bg-accent data-[active=true]:hover:bg-accent data-[active=true]:bg-accent/50 data-[active=true]:text-accent-foreground focus:bg-accent focus:text-accent-foreground [&amp;_svg:not([class*=&#x27;text-&#x27;])]:text-muted-foreground flex-col p-2 outline-none focus-visible:outline-1 [&amp;_svg:not([class*=&#x27;size-&#x27;])]:size-4 inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent hover:text-accent-foreground h-9 px-4 py-2" data-radix-collection-item="" href="https://calendar.google.com/calendar/u/0/appointments/AcZssZ3tWQd0YSITufTZaAitKeE6Ln2NBOouN5Qlop4=?gv=true">Calendar</a></li></ul></div></nav></nav><div class="flex items-center justify-end gap-3 text-muted-foreground"><button class="gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent py-2 h-9 w-9 px-0 flex items-center justify-center text-muted-foreground hover:text-foreground bg-transparent border-border shadow-none" aria-label="Open search (press Control or Command and K)"><i class="fa-jelly fa-search text-sm" aria-hidden="true"></i><span class="sr-only">Open search (press Control or Command and K)</span></button><div class="flex items-center gap-3"><button class="inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent w-9 px-3 h-9 text-muted-foreground hover:text-foreground shadow-none" type="button" id="radix-_R_6t4anpfalb_" aria-haspopup="menu" aria-expanded="false" data-state="closed" data-slot="dropdown-menu-trigger"><i class="fa-jelly fa-sun text-sm scale-100 rotate-0 transition-all dark:scale-0 dark:-rotate-90" aria-hidden="true"></i><i class="fa-jelly fa-moon absolute text-sm scale-0 rotate-90 transition-all dark:scale-100 dark:rotate-0" aria-hidden="true"></i><span class="sr-only">Toggle theme</span></button><a target="_blank" rel="noopener" class="inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 border border-input bg-background hover:bg-accent hover:text-accent-foreground py-2 gap-2 px-3 h-9 shadow-none" href="https://hashnode.com"><i class="text-sm fa-jelly fa-pencil" aria-hidden="true"></i>Write</a></div></div></div></div></header><div data-slot="dialog-header" class="flex flex-col gap-2 text-center sm:text-left sr-only"><h2 id="radix-_R_8anpfalbH1_" data-slot="dialog-title" class="text-lg leading-none font-semibold">Command Palette</h2><p id="radix-_R_8anpfalbH2_" data-slot="dialog-description" class="text-muted-foreground text-sm">Search for a command to run...</p></div><section class="py-8"><div class="px-4 sm:px-8"><div class="w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="grid gap-5 max-w-full"><article class="w-full max-w-full rounded-xl border bg-card text-card-foreground"><div class="px-5 pt-12 pb-8 sm:px-8"><header class="space-y-5"><div class="space-y-4 text-center max-w-4xl mx-auto"><h1 class="text-2xl sm:text-3xl md:text-4xl lg:text-5xl font-bold leading-tight tracking-tight text-balance text-foreground">Automated telemetry capture via Python bytecode modification</h1><p class="text-lg sm:text-xl leading-normal font-normal text-muted-foreground text-balance">The pursuit of capturing application telemetry automatically</p><div class="flex flex-wrap items-center justify-center gap-3 text-sm text-muted-foreground"><div class="flex items-center gap-2"><span>Published</span><time dateTime="2024-06-25T01:18:44.170Z">June 25, 2024</time></div><span class="text-muted-foreground">•</span><span>15<!-- --> min read</span><span class="text-muted-foreground">•</span><a href="/automated-telemetry-capture-via-python-bytecode-modification.md" target="_blank" rel="noopener" class="inline-flex items-center gap-2 text-muted-foreground transition-colors hover:text-foreground"><i class="fa-brands fa-markdown" aria-hidden="true"></i>View as Markdown</a></div></div></header></div><div class="w-full"><img alt="Automated telemetry capture via Python bytecode modification" width="1200" height="750" decoding="async" data-nimg="1" class="w-full h-auto" style="color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg"/></div><div class="px-5 py-12 sm:px-8 space-y-8"><div class="max-w-2xl mx-auto"><div class="flex flex-wrap items-center gap-5"><div class="flex items-center gap-5"><div class="flex gap-3 items-start"><a target="_blank" rel="noopener" href="https://hashnode.com/@jaywhy13"><span class="relative flex shrink-0 overflow-hidden rounded-full h-10 w-10 border border-border transition-opacity hover:opacity-80"><span class="flex h-full w-full items-center justify-center rounded-full bg-muted text-muted-foreground font-semibold text-lg">J</span></span></a><div class="min-w-0"><div class="flex items-center gap-2"><a target="_blank" rel="noopener" class="font-medium text-foreground hover:text-primary transition-colors" href="https://hashnode.com/@jaywhy13">Jean-Mark Wright</a><div class="flex items-center gap-1.5 text-sm"><a target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground hover:text-foreground transition-colors" aria-label="X (Twitter)" href="https://twitter.com/kramnaej"><i class="fa-brands fa-x-twitter" aria-hidden="true"></i></a><a target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground hover:text-foreground transition-colors" aria-label="LinkedIn" href="https://www.linkedin.com/in/jean-mark-wright/"><i class="fa-brands fa-linkedin" aria-hidden="true"></i></a></div></div><div class="text-sm text-muted-foreground line-clamp-2"><p>Jean-Mark is an articulate, customer-focused, visionary with extensive industry experience encompassing front and back end development in all phases of software development. He lives "outside the box" in pursuit of architectural elegance and relevant, tangible solutions. He is a strong believer that team empowerment fosters genius and is versed at working solo.</p>
</div></div></div></div></div></div><div class="relative"><aside class="hidden xl:block absolute top-0 left-0 h-full"><nav class="sticky top-22 w-44"><div class="hidden xl:block group/toc relative"><div class="grid"><div class="[grid-area:1/1] opacity-0 group-hover/toc:opacity-100 transition-opacity duration-200 max-h-[calc(100vh-3rem)] overflow-y-auto"><p class="text-sm font-semibold uppercase tracking-wide text-muted-foreground mb-3">On this page</p><nav aria-label="On this page"><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-introduction">Introduction</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-motivation">Motivation</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-step-0-the-existing-manual-process">Step 0 - The existing, manual process</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-step-1-capture-variables-in-a-method">Step 1 - Capture variables in a method</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-step-2-capture-the-locals-from-functions-we-call">Step 2 - Capture the locals from functions we call</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-step-3-dont-touch-the-code">Step 3 - Don&#x27;t touch the code</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-step-4-byte-off-more-than-you-can-chew">Step 4 - Byte off more than you can chew</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-step-5-choke">Step 5 - Choke</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-conclusion">Conclusion</a></nav></div><div class="[grid-area:1/1] relative flex flex-col gap-px transition-opacity duration-200 group-hover/toc:opacity-0 pointer-events-none self-start max-h-[calc(100vh-3rem)] overflow-hidden pl-3"><div class="absolute left-0 w-0.5 rounded-full bg-foreground transition-all duration-300 opacity-0" style="transform:translateY(0px);height:12px"></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:12px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:10px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:37px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:38px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:48px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:29px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:40px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:14px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:10px"></div></div></div></div></div></nav></aside><div class="max-w-2xl mx-auto min-w-0 contain-[inline-size]"><div class="prose dark:prose-invert prose-pre:bg-foreground prose-pre:text-background prose-pre:border prose-pre:border-border prose-pre:rounded prose-pre:px-5 prose-pre:py-4 dark:prose-pre:bg-muted dark:prose-pre:text-foreground dark:prose-pre:border-border prose-pre:overflow-x-auto max-w-none text-base sm:text-lg [&amp;&gt;div&gt;p:first-child]:mt-0 [&amp;&gt;div&gt;p:first-child]:pt-0 min-w-0 wrap-break-word [&amp;_a]:break-all **:max-w-full"><div><h2 id="heading-introduction">Introduction</h2>
<p>Ever heard the expression "Don't <strong>byte</strong> off more than you can chew"? Today, I've got one of those stories about our attempt to capture telemetry automatically using Python by introducing hooks at the <em>bytecode</em> level. Our company has several teams managing 20+ microservices and thousands of endpoint. We have a successful manual telemetry capture pattern that significantly reduces incident investigation time by enriching and attaching telemetry to our tools like DataDog, Sentry, and Sumo. However, it's too time-consuming for widespread adoption across all teams. Our team implemented this pattern efficiently, but replicating it manually for other teams would take years! This lead us to explore and eventually <em>abandon</em> an overly complex automated bytecode capture approach.</p>
<p>Below is the story of that journey.</p>
<h2 id="heading-motivation">Motivation</h2>
<p>Ultimately we want to increase our business uptime by reducing incident recovery time. We achieve this by enhancing our production experience through access to high quality telemetry for investigation. We're aiming to replicate a local debugger in production. Enhancing system state visibility accelerates our investigations, which reduces disruption time. If we're able to clearly see system state when looking at a trace, or an error, or log, we'll drastically reduce remediation time. High quality telemetry provides a solid base for hypotheses creation and testing.</p>
<p>We've had successes that confirm that this approach works too. We had an incident where we inadvertently gave <a target="_blank" href="https://jaywhy13.hashnode.dev/solving-like-sherlock-a-15-minute-case-with-observability" rel="noopener noreferrer nofollow ugc">free access</a> to some premium features. We got to our root cause for that issue in less than 15 minutes thanks to the tooling improvements. In that incident, we zoomed out from an individual trace and aggregated over a suspicious property which confirmed our theory. We had another incident where we fixed a broken Kafka worker and it caused data corruption for a running migration. Again, examining traces and inspecting system state was critical for understanding issues quickly.</p>
<h2 id="heading-step-0-the-existing-manual-process">Step 0 - The existing, manual process</h2>
<p>Let's take a look at what our manual process looks like. Below I'm showing the code for a <code>SubscriptionAPI</code>. This represents the highest component in our architecture. An API typically calls a service, then a repository for data operations. You'll see there's a <code>Context</code> (alias for a Python dictionary) that we start adding information to. That's us collecting telemetry. Below we're adding the input values to the <code>Context</code> like the <code>api</code>, <code>user_id</code>, <code>business_id</code>, <code>plan_id</code> (the subscription's plan) and eventually the <code>serialized_subscription</code> we generate as the API output. The <code>Context</code> is then passed down the hierarchy to all other calls. In this scenario, the <code>SubscriptionAPI</code> calls the <code>SubscriptionService.create_subscription</code> method. It passes the <code>ctx</code> to that method so we can continue to populate it.</p>
<pre><code class="lang-python"><span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">SubscriptionAPI</span>:</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">create_subscription</span>(<span class="hljs-params">
self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID
</span>):</span>
<span class="hljs-comment"># Start capturing telemetry (using a dictionary "ctx")</span>
ctx[<span class="hljs-string">"api"</span>] = <span class="hljs-string">"create_subscription"</span>
ctx[<span class="hljs-string">"user_id"</span>] = str(user_id)
ctx[<span class="hljs-string">"plan_id"</span>] = str(plan_id)
ctx[<span class="hljs-string">"business_id"</span>] = str(business_id)
<span class="hljs-comment"># Call the service to create a subscription</span>
subscription = SubscriptionService().create_subscription(
ctx, plan_id=plan_id, user_id=user_id, business_id=business_id
)
<span class="hljs-comment"># Serialize the subscription to JSON for return</span>
serialized_subscription = {
<span class="hljs-string">"id"</span>: str(subscription.id),
<span class="hljs-string">"plan_id"</span>: str(subscription.plan_id),
<span class="hljs-string">"user_id"</span>: str(subscription.user_id),
<span class="hljs-string">"business_id"</span>: str(subscription.business_id),
<span class="hljs-string">"status"</span>: subscription.status.value,
}
ctx[<span class="hljs-string">"serialized_subscription"</span>] = serialized_subscription
<span class="hljs-keyword">return</span> serialized_subscription
</code></pre>
<p>As we populate the <code>ctx</code> in each layer of the application, we end up with a rich description of our current system state.</p>
<p>Below is an example of us invoke the <code>SubscriptionAPI</code> and then examining the <code>Context</code>.</p>
<pre><code class="lang-python"><span class="hljs-comment"># Declare variables (plan_id, user_id, etc...)</span>
ctx = {}
subscription_api = SubscriptionAPI()
subscription_api.create_subscription(
ctx, plan_id, user_id, business_id)
print(ctx)
</code></pre>
<p>The output of the <code>Context</code> looks like this:</p>
<pre><code class="lang-json">{<span class="hljs-attr">"api"</span>: <span class="hljs-string">"create_subscription"</span>,
<span class="hljs-attr">"business_id"</span>: <span class="hljs-string">"bc7464cc-dd17-481e-88a4-93ba5711f2a1"</span>,
<span class="hljs-attr">"plan_id"</span>: <span class="hljs-string">"43a3ec11-ba1d-4fff-a958-f3945fc46a36"</span>,
<span class="hljs-attr">"serialized_subscription"</span>: {<span class="hljs-attr">"business_id"</span>: <span class="hljs-string">"bc7464cc-dd17-481e-88a4-93ba5711f2a1"</span>,
<span class="hljs-attr">"id"</span>: <span class="hljs-string">"f2dc517e-5c9c-4f55-8a26-ef43dd7bc0ef"</span>,
<span class="hljs-attr">"plan_id"</span>: <span class="hljs-string">"43a3ec11-ba1d-4fff-a958-f3945fc46a36"</span>,
<span class="hljs-attr">"status"</span>: <span class="hljs-string">"active"</span>,
<span class="hljs-attr">"user_id"</span>: <span class="hljs-string">"61346915-3c35-4ba4-87fd-7d092053c940"</span>},
<span class="hljs-attr">"status"</span>: <span class="hljs-string">"active"</span>,
<span class="hljs-attr">"user_id"</span>: <span class="hljs-string">"61346915-3c35-4ba4-87fd-7d092053c940"</span>}
</code></pre>
<p>This manual capture approach works well when you're starting fresh and you establish this as a team practice. However, when you've got thousands of endpoints scattered across multiple services without the pattern, we need an automated approach.</p>
<p>Let's dive into the steps we took on the journey to automated telemetry capture.</p>
<h2 id="heading-step-1-capture-variables-in-a-method">Step 1 - Capture variables in a method</h2>
<p>The first step is understanding how we capture all the variables in a method. Once we can capture variables in one method, we can extend this approach to the rest of the architecture stack. Python provides a facility for accomplishing this. We can use <code>inspect.stack()</code> to get local variables from a function. The stack stores variables for each function being called. Python creates a frame for each function being called. Let's see how this works in practice.</p>
<pre><code class="lang-python"><span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">SubscriptionAPI</span>:</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">create_subscription</span>(<span class="hljs-params">
self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID
</span>):</span>
<span class="hljs-comment"># Call the service to create a subscription</span>
subscription = SubscriptionService().create_subscription(
ctx, plan_id=plan_id, user_id=user_id, business_id=business_id
)
<span class="hljs-comment"># Serialize the subscription to JSON for return</span>
serialized_subscription = {...}
<span class="hljs-comment"># Capture all the variables in the method</span>
stack = inspect.stack()
local_variables = stack[<span class="hljs-number">0</span>].frame.f_locals
<span class="hljs-comment"># Capture all the local variables in the context</span>
ctx.update(local_variables)
<span class="hljs-keyword">return</span> serialized_subscription
</code></pre>
<p>Near the end of our <code>create_subscription</code> function, we're accessing the <code>stack</code> and printing the locals in the first frame (i.e. at index <code>0</code>). This gives us all the local variables in the current function (i.e. <code>create_subscription</code>). Every time Python calls a function, it creates a frame for it and stores the locals.</p>
<p>If we run the <code>create_subscription</code> method and examine the <code>ctx</code>, it looks like this:</p>
<pre><code class="lang-python">{<span class="hljs-string">'business_id'</span>: UUID(<span class="hljs-string">'bbe18a98-d565-4423-a13a-1c17b38b003d'</span>),
<span class="hljs-string">'plan_id'</span>: UUID(<span class="hljs-string">'81df53ad-2159-4566-b43f-3c24d55d9d3e'</span>),
<span class="hljs-string">'serialized_subscription'</span>: {<span class="hljs-string">'business_id'</span>: <span class="hljs-string">'bbe18a98-d565-4423-a13a-1c17b38b003d'</span>,
<span class="hljs-string">'id'</span>: <span class="hljs-string">'977ac956-496f-4b54-8627-a0fd4fc3e9df'</span>,
<span class="hljs-string">'plan_id'</span>: <span class="hljs-string">'81df53ad-2159-4566-b43f-3c24d55d9d3e'</span>,
<span class="hljs-string">'status'</span>: <span class="hljs-string">'active'</span>,
<span class="hljs-string">'user_id'</span>: <span class="hljs-string">'ade39122-70b9-47f8-94e9-6ab4926e7398'</span>},
<span class="hljs-string">'subscription'</span>: Subscription(id=UUID(<span class="hljs-string">'977ac956-496f-4b54-8627-a0fd4fc3e9df'</span>),
plan_id=UUID(<span class="hljs-string">'81df53ad-2159-4566-b43f-3c24d55d9d3e'</span>),
user_id=UUID(<span class="hljs-string">'ade39122-70b9-47f8-94e9-6ab4926e7398'</span>),
business_id=UUID(<span class="hljs-string">'bbe18a98-d565-4423-a13a-1c17b38b003d'</span>),
status=&lt;SubscriptionStatus.ACTIVE: <span class="hljs-string">'active'</span>&gt;),
<span class="hljs-string">'user_id'</span>: UUID(<span class="hljs-string">'ade39122-70b9-47f8-94e9-6ab4926e7398'</span>)}
</code></pre>
<p>As you see from the output, we've captured all the fields from the <code>create_subscription</code> method.</p>
<p>Let's add a visual to concretize the concept.</p>
<p><img src="https://cdn.hashnode.com/res/hashnode/image/upload/v1718937071544/663c1e0b-b193-4233-b1d7-53e58492e5b1.png" alt class="image--center mx-auto" /></p>
<p>When we call <code>SubscriptionAPI.create_subscription</code>, Python pushes a frame unto the stack. That frame contains the locals of our method.</p>
<p>Sounds good! We're making progress.</p>
<h2 id="heading-step-2-capture-the-locals-from-functions-we-call">Step 2 - Capture the locals from functions we call</h2>
<p>We don't only need to capture the locals from the <code>SubscriptionAPI</code> call, but all the other components (e.g. services, repositories, clients) it calls to satisfy our request. The <code>SubscriptionAPI</code> will call the <code>SubscriptionService</code> to create the subscription. The <code>SubscriptionService</code> will in turn, call the repository (<code>SubscriptionRepository</code>) to convince the database to create the data.</p>
<p>Let's look at a quick sequence diagram to concretize the picture in our minds.</p>
<p><img src="https://cdn.hashnode.com/res/hashnode/image/upload/v1718937505858/36b51f30-f97b-4445-a337-6e65e921caf2.png" alt class="image--center mx-auto" /></p>
<p>Thankfully, Python can help us here too! Python captures all the variables for all the functions that get called. We learned earlier that each time Python invokes a function, it creates a stack frame with the locals. When a function calls another function, Python creates a new stack frame for the new function and its locals. So as functions are calling other functions, Python pushes new frames unto the stack. As such, we just need to loop through each frame to capture all the variables.</p>
<p>Let's look at the code for each of the components. They're mostly passthroughs.</p>
<pre><code class="lang-python"><span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">SubscriptionRepository</span>:</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">create_subscription</span>(<span class="hljs-params">
self,
ctx: Context,
plan_id: UUID,
user_id: UUID,
business_id: UUID,
status: SubscriptionStatus,
</span>):</span>
<span class="hljs-comment"># Pretend we're calling the ORM to create the subscription</span>
<span class="hljs-comment"># Serialize the output as a Subcription value object</span>
<span class="hljs-keyword">return</span> Subscription(...)
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">SubscriptionService</span>:</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">create_subscription</span>(<span class="hljs-params">
self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID
</span>) -&gt; Subscription:</span>
status = SubscriptionStatus.ACTIVE
<span class="hljs-keyword">return</span> SubscriptionRepository().create_subscription(...)
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">SubscriptionAPI</span>:</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">create_subscription</span>(<span class="hljs-params">
self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID
</span>):</span>
<span class="hljs-comment"># Call the service to create a subscription</span>
subscription = SubscriptionService().create_subscription(...)
<span class="hljs-comment"># Serialize the subscription to JSON for return</span>
serialized_subscription = {...}
<span class="hljs-keyword">return</span> serialized_subscription
</code></pre>
<p>If we were to look at the stack, when the <code>SubscriptionRepository.create_subscription</code> function is being interpreted, the stack would contain frames for the calling components (i.e. API and service).</p>
<p><img src="https://cdn.hashnode.com/res/hashnode/image/upload/v1718938824076/1699c24c-4493-4a4d-9d7f-98408a549038.png" alt class="image--center mx-auto" /></p>
<p>First, there's a frame for the current method (i.e. <code>SubscriptionRepository.create_subscription</code>) with all its variables. It was called by <code>SubscriptionService.create_subscription</code>, so there's a stack for that. Finally, the service was called by <code>SubscriptionAPI.create_subscription</code> so there's a frame for that one as well.</p>
<p>Let's look at how we'd achieve that.</p>
<pre><code class="lang-python"><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_variables_from_all_calls</span>() -&gt; dict:</span>
<span class="hljs-string">"""Get all the variables in the method that called us."""</span>
local_variables = {}
stack = inspect.stack()
<span class="hljs-comment"># We don't want the current frame, that's this function</span>
all_frames_except_this_one = stack[<span class="hljs-number">1</span>:]
<span class="hljs-keyword">for</span> frame_information <span class="hljs-keyword">in</span> all_frames_except_this_one:
frame = frame_information.frame
local_variables.update(frame.f_locals)
<span class="hljs-keyword">return</span> local_variables
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">SubscriptionRepository</span>:</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">create_subscription</span>(<span class="hljs-params">
self,
ctx: Context,
plan_id: UUID,
user_id: UUID,
business_id: UUID,
status: SubscriptionStatus,
</span>):</span>
<span class="hljs-comment"># Call the ORM to create the subscription</span>
<span class="hljs-comment"># Update the context with variables from all calls in the stack</span>
ctx.update(get_variables_from_all_calls())
<span class="hljs-comment"># Serialize the output as a Subcription object</span>
<span class="hljs-keyword">return</span> Subscription(...)
</code></pre>
<p>In this example, I'm introducing a helper (<code>get_variables_from_all_calls</code>) to move the logic outside of our main function. Since the repository will call this method, that means Python will add a frame to the stack for this helper function as well. As such, we need to get all the frames except the current one. This explains why we're only capturing frames from index <code>1</code> and beyond. Finally, in <code>SubscriptionRepository.create_subscription</code> we're calling <code>get_variables_from_all_calls</code> to get all the variables for us.</p>
<p>When we run the code, the <code>ctx</code> has the following contents:</p>
<pre><code class="lang-python">{<span class="hljs-string">'business_id'</span>: UUID(<span class="hljs-string">'3498a3df-5001-481c-b0b8-a007609fb93c'</span>),
<span class="hljs-string">'plan_id'</span>: UUID(<span class="hljs-string">'fefbbafd-0f59-49bb-9255-3c09eef38fec'</span>),
<span class="hljs-string">'status'</span>: &lt;SubscriptionStatus.ACTIVE: <span class="hljs-string">'active'</span>&gt;,
<span class="hljs-string">'user_id'</span>: UUID(<span class="hljs-string">'de19ca36-67c5-48eb-80c8-70b0db9575ac'</span>)}
</code></pre>
<p>If you're paying close attention, you'll notice this list is a little shorter than the last time we printed out the <code>Context</code>. In particular, this is missing the <code>serialized_subscription</code> and <code>subscription</code> object found in the <code>SubscriptionAPI.create_subscription</code> method. That happens because when we're capturing the variables, those don't exist yet.</p>
<p>See below if you're not convinced.</p>
<pre><code class="lang-python"><span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">SubscriptionAPI</span>:</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">create_subscription</span>(<span class="hljs-params">
self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID
</span>):</span>
<span class="hljs-comment"># The subscription variable is defined AFTER the call returns</span>
subscription = SubscriptionService().create_subscription(...)
<span class="hljs-comment"># Similarly, this is also defined AFTER the call returns</span>
serialized_subscription = {...}
<span class="hljs-keyword">return</span> serialized_subscription
</code></pre>
<p>We'll keep going! That's a small problem in the grand scheme of things. As you'll see in the next step... we've got bigger problems.</p>
<h2 id="heading-step-3-dont-touch-the-code">Step 3 - Don't touch the code</h2>
<p>One of our primary goals was to enable telemetry without manual effort. If we need to be adding a bunch of <code>get_variables_from_all_calls</code> calls all over our code, that simply won't work. It's less work than manually adding every variable, but still too much effort. Additionally, we don't want folks to modify their code to include these calls. We want these utilities to be mostly invisible but providing all the benefits necessary.</p>
<p>Let's iterate and see what we can accomplish...</p>
<p>What if we tried capturing the stack variables after the we made the function call.</p>
<pre><code class="lang-python">subscription_api = SubscriptionAPI()
ctx = {}
subscription_api.create_subscription(
ctx, plan_id, user_id, business_id)
<span class="hljs-comment"># Capture all variables</span>
get_variables_from_all_calls()
</code></pre>
<p>If we print out the context of the <code>Context</code>, we'd get the following:</p>
<pre><code class="lang-python">{}
</code></pre>
<p>The <code>Context</code> is empty. Yeah, that's right... it's completely empty.</p>
<p>Here comes another important detail I omitted before...</p>
<blockquote>
<p><strong>Once a function is done executing, Python pops its frame from the stack.</strong></p>
</blockquote>
<p>So... if you were to look at the stack at this point, it'd be empty (technically there's a "module" frame, but I've omitted that for brevity). There are no other frames on the stack, because after the function calls have exited, the frames associated with each call are completely gone. All of the previous approaches worked because we were capturing variables while we were <em>in</em> the call stack. Once those functions have exited we're no longer in the call stack and don't have access to those variables.</p>
<p>So we need a different approach. We need to capture variables while the relevant frames still exist on the stack. But how...</p>
<h2 id="heading-step-4-byte-off-more-than-you-can-chew">Step 4 - Byte off more than you can chew</h2>
<p>It turns out, there's a way we can utilize the same approach, without modifying the code folks would write. We can introduce our capturing logic, at a lower-level... much lower. When we write Python code, it gets compiled into bytecode, which is then run by the interpreter.</p>
<p><img src="https://cdn.hashnode.com/res/hashnode/image/upload/v1719277675826/ba7ff392-856d-4314-b83b-2cfa8ad75c03.png" alt class="image--center mx-auto" /></p>
<p>We could <em>modify</em> the bytecode to accomplish the same objective. Python bytecode looks very different from Python. It's a sequence of instructions for the Python virtual machine. These instructions manipulate the internal stacks that Python uses. By modifying the resulting bytecode we can inject our logic and have the compiler execute our logic and capture the variables.</p>
<p>To exactly replicate what we did at a higher level, we need to capture the stack just before the function returns. So, we'd need to parse the bytecode instructions, looking for a return statement and then inject our logic right before that. We'd insert our logic to call the <code>get_variables_from_all_calls</code> function which would extract the variables from the stack. Then finally we'd relace the function's bytecode with our patched version. Sounds simple enough, right?</p>
<p>There are lots of different bytecode instructions. The following are the ones we're interested in:</p>
<ul>
<li><p><code>LOAD_GLOBAL</code> - this instruction enables us to make our function available for calling. This instruction tells Python to look up an object in the global namespace and place it on the evaluation stack. We'll use this to make our function ready for calling.</p>
</li>
<li><p><code>LOAD_FAST</code> - this instruction enables us to provide arguments for a function call. This pushes a variable unto the stack for evaluation.</p>
</li>
<li><p><code>CALL_FUNCTION</code> - this instruction is used to execute our function call. With this instruction we can also provide a number of positional arguments that should be popped and passed to the function.</p>
</li>
<li><p><code>POP_TOP</code> - this instruction allows us to remove our function from the stack to continue the flow of the program. We'll use this to pop our function after we've called it.</p>
</li>
</ul>
<p>Next, I need to explain an idiosyncrasy we have to be familiar with to make this all work. Sometimes instructions spans multiple lines. We only need to add our patching instructions once. As such, we need to keep track of whether we've patched a line already so we don't do it multiple times.</p>
<p>Armed with that knowledge let's examine the function that does the patching.</p>
<pre><code class="lang-python"><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">patch_function_to_capture_stack_variables</span>(<span class="hljs-params">func</span>):</span>
<span class="hljs-comment"># Get the bytecode that we plan to patch</span>
code = func.__code__
bytecode = Bytecode.from_code(code)
<span class="hljs-comment"># Get a list of all line numbers for return instructions</span>
<span class="hljs-comment"># Recall instructions can span multiple lines</span>
return_line_numbers = [
instruction.lineno
<span class="hljs-keyword">for</span> instruction <span class="hljs-keyword">in</span> bytecode
<span class="hljs-keyword">if</span> instruction.name == <span class="hljs-string">"RETURN_VALUE"</span>
]
<span class="hljs-comment"># For each return instruction, we want to insert a call to capture_stack</span>
<span class="hljs-keyword">for</span> return_line_number <span class="hljs-keyword">in</span> return_line_numbers:
<span class="hljs-comment"># construct the instructions we need</span>
<span class="hljs-comment"># Add our function to the stack</span>
add_capture_stack_function_to_the_stack = Instr(<span class="hljs-string">"LOAD_GLOBAL"</span>, <span class="hljs-string">"get_variables_from_all_calls"</span>)
<span class="hljs-comment"># Add the ctx as an argument for the function</span>
add_ctx_as_an_argument = Instr(<span class="hljs-string">"LOAD_FAST"</span>, <span class="hljs-string">"ctx"</span>)
<span class="hljs-comment"># Call our function and pop 1 arguemnt (ctx) to be passed to the function</span>
call_the_capture_stack_function = Instr(<span class="hljs-string">"CALL_FUNCTION"</span>, <span class="hljs-number">1</span>)
<span class="hljs-comment"># Remove our function to continue normal processing</span>
remove_the_capture_stack_function_from_the_stack = Instr(<span class="hljs-string">"POP_TOP"</span>)
<span class="hljs-comment"># Put the instructions all together</span>
catpure_stack_instructions = [
add_capture_stack_function_to_the_stack,
add_ctx_as_an_argument,
call_the_capture_stack_function,
remove_the_capture_stack_function_from_the_stack,
]
<span class="hljs-keyword">for</span> i, instruction <span class="hljs-keyword">in</span> enumerate(bytecode):
<span class="hljs-keyword">if</span> instruction.lineno == return_line_number:
bytecode[i:i] = catpure_stack_instructions
<span class="hljs-comment"># There could be other instructions on the same line</span>
<span class="hljs-comment"># We don't want to insert the new instructions multiple times</span>
<span class="hljs-comment"># So break out of the loop</span>
<span class="hljs-keyword">break</span>
updated_code = bytecode.to_code()
func.__code__ = updated_code
</code></pre>
<p>As shown in the code above, we're getting the bytecode, adding instructions to call our function and then applying the patch. We start by getting the <code>__code__</code> object of the function. That's the bytecode! We're using the <code>bytecode</code> library to simplify going to and from bytecode and construction of instructions. Next, we find all the lines that have return instructions. Finally, we construct our new instructions and add them to the bytecode.</p>
<p>Finally, we patch the <code>SubscriptionRepository.create_subscription</code> function so it'll call our function.</p>
<pre><code class="lang-python">patch_function_to_capture_stack_variables(
SubscriptionRepository.create_subscription)
subscription_api = SubscriptionAPI()
subscription_api.create_subscription(
ctx, plan_id, user_id, business_id)
print(ctx)
</code></pre>
<p>Running this code, gets us the following output:</p>
<pre><code class="lang-python"> {<span class="hljs-string">'business_id'</span>: UUID(<span class="hljs-string">'276dac6c-68ad-48bf-8a55-ac7ee6df64a5'</span>),
<span class="hljs-string">'plan_id'</span>: UUID(<span class="hljs-string">'e72b5c80-63d7-4c5a-a2a9-d4b5eef937e0'</span>),
<span class="hljs-string">'status'</span>: &lt;SubscriptionStatus.ACTIVE: <span class="hljs-string">'active'</span>&gt;,
<span class="hljs-string">'user_id'</span>: UUID(<span class="hljs-string">'946bf774-e95a-488d-b5b5-55a255aa5d2a'</span>)}
</code></pre>
<p>Success! Without asking anyone to modify their code, we were able to capture the local variables from the stack! This provides significant gains as we're able to reduce the manual effort involved. It's leaps and bounds further than where we started.</p>
<p>You'll notice though that it's the same output as step 3. It still has the shortcoming where it doesn't capture variables that haven't been defined yet. There are ways to overcome this that don't take too much effort and don't re-introduce manual effort. Anyway... that's a story for another time.</p>
<h2 id="heading-step-5-choke">Step 5 - Choke</h2>
<p>I've had to acknowledge that despite the energy rush this investigation brought, its complexity is well beyond our company's maintenance capacity. None of us are experts in the topic of bytecode. Theoretically, we're not doing much -- we're simply hooking into the code at a lower level. We wanted to avoid folks littering their code with our hooking logic, so we moved down an abstraction layer or two. That descent was unfortunately dangerously close to 6 feet under. Writing our Observability tooling hooks in bytecode would be suicidal for our tooling's life.</p>
<p>What's worse, is that the implementation I've described thus far is woefully inadequate. We'd need to make several improvements and additions to <em>begin</em> the journey to production. We'd need some more bytecode instructions to import our library from elsewhere (in the script it's defined in the same file). Discover and account for additional idiosyncrasies of bytecode. We haven't addressed the multiple ways that functions can exit (e.g. early returns, unexpected exceptions and try-catch scenarios). Additionally, we would want to make our code more robust. We don't want failures in our library to cascade to the rest of the application. We'd probably need to catch all failures in our calls and properly log them (e.g. Sentry, SumoLogic, DataDog, etc...), or capture with metrics. Then, finally, we haven't thought about optimization or security. Who knows what challenges lie in that space!</p>
<p>And so, we've started investigating alternatives. We've got some basic ideas we're toying around. One idea is to reduce the complexity, by only capturing function inputs and outputs. We could use a decorator-based approach to wrap public methods of all our components and capture the variables. We can implement our wrapping logic in a metaclass that we use to initialize our different components (e.g. services, repositories, clients, etc...). It's early days and we're still exploring things. I'll return to blog about our success (let's hope!) once we've found a working solution.</p>
<h2 id="heading-conclusion">Conclusion</h2>
<p>In our attempt to automate telemetry capture using Python, we explored the ambitious idea of introducing hooks at the bytecode level to automate this process. While our manual approach of enriching telemetry within our services has proven effective in reducing incident investigation time, scaling this across multiple teams and microservices was impractical. Our journey involved understanding Python's stack and bytecode intricacies to capture local variables across function calls. Despite the surge and adrenaline of this approach, the unavoidable complexity and maintenance challenges, lead us to abandon this method. We're now considering simpler, more maintainable alternatives like using decorators to capture function inputs and outputs.</p>
<p>Thanks for reading! If you've got thoughts on this topic, I'd love to hear it! Feel free to reach out to me on <a target="_blank" href="https://x.com/kramnaej" rel="noopener noreferrer nofollow ugc">Twitter</a> or <a target="_blank" href="https://www.linkedin.com/in/jean-mark-wright/" rel="noopener noreferrer nofollow ugc">LinkedIn</a>!</p>
<p>Also, many thanks to the <a target="_blank" href="https://github.com/DataDog/datadogpy" rel="noopener noreferrer nofollow ugc">DataDog library</a> I reverse-engineered to get an idea of how this could be accomplished.</p>
<p>Here are a few resources that were helpful, if you're looking to learn more about bytecode.</p>
<ul>
<li><p><a target="_blank" href="https://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1" rel="noopener noreferrer nofollow ugc">Demystifying Python Bytecode: A Guide to Understanding and Analyzing Code Execution</a></p>
</li>
<li><p><a target="_blank" href="https://github.com/DataDog/datadogpy" rel="noopener noreferrer nofollow ugc">DataDog Python Library</a></p>
</li>
</ul>
<p><a target="_blank" href="https://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1" rel="noopener noreferrer nofollow ugc">https://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1</a></p>
</div></div></div></div><div class="max-w-2xl mx-auto border-t border-border pt-8 space-y-5"><div class="flex flex-wrap items-center gap-2"><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/python">#<!-- -->python</a><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/bytecode">#<!-- -->bytecode</a><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/observability">#<!-- -->observability</a><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/telemetry">#<!-- -->telemetry</a><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/microservices">#<!-- -->microservices</a><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/instrumentation">#<!-- -->instrumentation</a></div></div></div></article></div></div></div></section><section class="pb-8"><div class="px-4 sm:px-8"><div class="w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="max-w-4xl w-full mx-auto"><div class="rounded-xl border bg-card text-card-foreground p-5 sm:p-8"><section class="space-y-5"><div class="flex items-center justify-between"><h2 class="text-lg font-bold tracking-tight">Comments</h2><a target="_blank" rel="noopener" class="inline-flex items-center gap-2 rounded-md border border-border px-3 py-1.5 text-sm font-medium text-foreground transition hover:border-primary hover:text-primary" href="https://hashnode.com/posts/automated-telemetry-capture-via-python-bytecode-modification/667a1af4d2255d267d58e1df"><i class="fa-regular fa-comment" aria-hidden="true"></i>Join the discussion</a></div><p class="text-sm text-muted-foreground py-5 text-center">No comments yet.<!-- --> <a target="_blank" rel="noopener" class="text-primary hover:underline" href="https://hashnode.com/posts/automated-telemetry-capture-via-python-bytecode-modification/667a1af4d2255d267d58e1df">Be the first to comment.</a></p></section></div></div></div></div></section><section class="pb-8"><div class="px-4 sm:px-8"><div class="w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="max-w-4xl w-full mx-auto"><section class="space-y-6"><h2 class="text-2xl font-bold tracking-tight text-balance">More from this blog</h2><div class="flex flex-col gap-6"><article aria-labelledby="_R_panpfalb_" class="h-full"><a class="block h-full" href="/instrumenting-at-10-years-per-second"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_panpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Instrumenting at 10 years per second</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">How we made custom instrumentation blazing fast, simple, and data-centric</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Dec 23, 2025</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>8 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Instrumenting at 10 years per second" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg"/></div></div></div></div></a></article><article aria-labelledby="_R_19anpfalb_" class="h-full"><a class="block h-full" href="/on-shipping-smaller-prs"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_19anpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">On shipping smaller PRs</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">Concrete examples for how to ship smaller PRs</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Feb 17, 2025</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>8 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="On shipping smaller PRs" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/o9oQaOGpLz0/upload/9e19fbfd378d65c448f7233b2a273b79.jpeg"/></div></div></div></div></a></article><article aria-labelledby="_R_1panpfalb_" class="h-full"><a class="block h-full" href="/small-prs-ship-faster-qed"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_1panpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Small PRs ship faster - QED</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">My unrestrained passion for shipping smaller PRs constrained into a handful of benefits</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Dec 25, 2024</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>5 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Small PRs ship faster - QED" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/JzMS3RLSZn8/upload/a18ee8f82d0d56644ab82f6c1bead8b4.jpeg"/></div></div></div></div></a></article><article aria-labelledby="_R_29anpfalb_" class="h-full"><a class="block h-full" href="/order-matters-making-a-compound-index-50x-faster"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_29anpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Order matters - making a compound index 50x faster</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">A story of how we made a compound index 50x faster by re-organizing the fields</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Oct 17, 2024</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>6 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Order matters - making a compound index 50x faster" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/upload/v1726107899370/04b1ac6d-7825-4a05-b5d0-0be78a2f49d9.png"/></div></div></div></div></a></article><article aria-labelledby="_R_2panpfalb_" class="h-full"><a class="block h-full" href="/dear-editor-we-need-better-database-observability"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_2panpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Dear Editor: We need better Database Observability</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">A call for better database observability, or awareness to a better reality</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Sep 12, 2024</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>8 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Dear Editor: We need better Database Observability" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/NvFkYV2ngOk/upload/2f4cbc39f32355a334d9119af2dd086c.jpeg"/></div></div></div></div></a></article><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 p-5 flex flex-col"><div class="flex flex-col items-center text-center gap-5"><div class="flex flex-col items-center gap-3"><div class="flex h-16 w-16 shrink-0 items-center justify-center rounded-full bg-muted"><i class="text-2xl fa-brands fa-hashnode text-foreground" aria-hidden="true"></i></div><div class="space-y-1"><p class="text-base font-semibold text-foreground leading-tight">Perspective Unspoken</p><p class="text-sm text-muted-foreground">19<!-- --> posts</p></div></div><div class="prose prose-sm dark:prose-invert max-w-xs mx-auto text-muted-foreground"><p>Welcome to "Perspective Unspoken", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.</p>
</div><div class="flex flex-wrap items-center justify-center gap-3"><a href="https://twitter.com/kramnaej" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="Twitter"><i class="fa-brands fa-x-twitter" aria-hidden="true"></i></a><a href="https://github.com/jaywhy13" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="GitHub"><i class="fa-brands fa-github" aria-hidden="true"></i></a><a href="https://www.linkedin.com/in/jean-mark-wright/" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="LinkedIn"><i class="fa-brands fa-linkedin" aria-hidden="true"></i></a><a href="https://hashnode.com/@jaywhy13" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="Hashnode"><i class="fa-brands fa-hashnode" aria-hidden="true"></i></a></div></div></div></div></section></div></div></div></section><script type="application/ld+json">{"@context":"https://schema.org","@type":"Article","url":"https://jaywhy13.hashnode.dev/automated-telemetry-capture-via-python-bytecode-modification","mainEntityOfPage":"https://jaywhy13.hashnode.dev/automated-telemetry-capture-via-python-bytecode-modification","headline":"Automated telemetry capture via Python bytecode modification","description":"Introduction\nEver heard the expression \"Don't byte off more than you can chew\"? Today, I've got one of those stories about our attempt to capture telemetry automatically using Python by introducing hooks at the bytecode level. Our company has several...","author":{"@type":"Person","name":"Jean-Mark Wright","url":"https://hashnode.com/@jaywhy13","image":"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg","sameAs":"https://x.com/kramnaej"},"publisher":{"@type":"Organization","name":"Perspective Unspoken","url":"https://jaywhy13.hashnode.dev"},"datePublished":"2024-06-25T01:18:44.170Z","dateModified":"2024-06-25T01:18:44.170Z","image":{"@type":"ImageObject","url":"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg"},"keywords":"Python, bytecode, observability, telemetry, Microservices, instrumentation"}</script><footer class="px-4 pt-8 pb-20 sm:px-8 text-muted-foreground" aria-label="Footer"><div class="w-full max-w-4xl mx-auto"><div class="flex flex-col items-center gap-4 text-center text-sm"><p class="font-medium"><time dateTime="2026"><span class="font-sans">©</span> <!-- -->2026</time> <!-- -->Perspective Unspoken</p><nav aria-label="Footer navigation"><ul class="flex flex-wrap justify-center gap-4 text-sm tracking-wide"><li><a class="transition hover:text-foreground" href="/archive">Archive</a></li><li><a class="transition hover:text-foreground" target="_blank" rel="noopener" href="https://hashnode.com/privacy-policy">Privacy</a></li><li><a class="transition hover:text-foreground" target="_blank" rel="noopener" href="https://hashnode.com/terms">Terms</a></li></ul></nav><div class="pt-8 flex justify-center gap-6"><a target="_blank" rel="noopener" class="flex items-center gap-2 text-sm transition hover:text-foreground" title="Sitemap" href="/sitemap.xml"><i class="fa-regular fa-sitemap"></i><span>Sitemap</span></a><a target="_blank" rel="noopener" class="flex items-center gap-2 text-sm transition hover:text-foreground" title="RSS Feed" href="/rss.xml"><i class="fa-regular fa-rss"></i><span>RSS</span></a></div><div class="pt-10 text-center"><a target="_blank" rel="noopener" aria-label="Visit Hashnode (opens in a new tab)" class="inline-flex items-center justify-center gap-2 whitespace-nowrap font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&amp;_svg]:pointer-events-none [&amp;_svg]:size-4 [&amp;_svg]:shrink-0 hover:bg-accent hover:text-accent-foreground h-10 rounded-md text-xl px-5 py-4" href="https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev&amp;utm_medium=referral&amp;utm_campaign=blog_footer_logo&amp;utm_content=logo"><svg fill="none" viewBox="0 0 282 39" aria-hidden="true" class="h-6 w-auto min-w-[9.5rem] text-muted-foreground transition hover:text-foreground"><path fill="currentColor" d="M2.68 13.032c-3.573 3.505-3.573 9.363 0 12.936L13.032 36.32c3.505 3.573 9.363 3.573 12.936 0L36.32 25.968c3.573-3.573 3.573-9.431 0-12.936L25.968 2.68c-3.573-3.573-9.431-3.573-12.936 0zm12.211 1.935c2.507-2.521 6.582-2.544 9.104-.038s2.544 6.582.038 9.104-6.582 2.544-9.104.038-2.544-6.582-.038-9.104"></path><path fill="currentColor" d="M268.379 39q-6.322 0-10.35-4.108-4.002-4.134-4.002-10.483 0-6.296 4.002-10.43 4-4.135 10.163-4.135 6.028 0 9.817 3.734 3.814 3.735 3.814 9.497v2.8h-24.248v-4.054h17.633q-.187-2.747-2.054-4.508-1.868-1.76-4.962-1.76-3.28-.001-5.388 2.374-2.108 2.373-2.108 6.375 0 4.188 2.348 6.695 2.347 2.481 5.895 2.481 1.68 0 2.934-.453a8.5 8.5 0 0 0 2.321-1.307q1.041-.854 2.268-2.614l4.694 2.96q-1.707 2.748-3.494 4.135a12.7 12.7 0 0 1-3.975 2.081q-2.187.72-5.308.72m-33.496 0q-5.575 0-9.283-4.028-3.681-4.029-3.681-10.484 0-6.43 3.761-10.51 3.76-4.108 9.47-4.108 2.8 0 5.175 1.2 2.4 1.174 3.628 3.148l.026-.08V0h7.016v38.2h-6.722v-4.722l.64.8-.694.133q-1.28 2.108-3.788 3.361Q237.924 39 234.883 39m1.627-6.002q3.308 0 5.495-2.428t2.188-6.135q0-3.762-2.214-6.109-2.214-2.374-5.469-2.374-3.28 0-5.522 2.347-2.214 2.348-2.214 6.163 0 3.76 2.214 6.162 2.214 2.374 5.522 2.374M205.811 39q-6.376 0-10.537-4.161-4.161-4.188-4.161-10.404 0-6.321 4.161-10.456 4.188-4.135 10.537-4.135 6.348 0 10.537 4.161 4.188 4.161 4.188 10.43 0 6.216-4.188 10.404Q212.186 39 205.811 39m0-6.162q3.307 0 5.575-2.374 2.267-2.374 2.267-6.055 0-3.707-2.294-6.056-2.267-2.347-5.548-2.347-3.308 0-5.575 2.374-2.268 2.348-2.268 6.029t2.241 6.055q2.268 2.374 5.602 2.374M162.586 38.2V10.644h6.802v5.015l-.8-.987h.906q1.308-2.187 3.762-3.494 2.48-1.335 5.228-1.334 4.615 0 7.256 2.8 2.667 2.776 2.667 7.603V38.2h-7.069V21.794q0-2.854-1.387-4.401t-4.134-1.547q-2.614 0-4.402 1.974-1.76 1.974-1.76 4.881V38.2zm-30.181 0V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.069V21.794q0-2.827-1.387-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.814 2.027-1.814 4.962v15.391zm-14.416.8q-2.882 0-5.015-.64-2.108-.614-3.761-1.84-1.654-1.228-2.668-2.695l-.987-1.467 4.695-3.174.773 1.067a9.1 9.1 0 0 0 1.788 1.814 8.2 8.2 0 0 0 2.32 1.2q1.308.4 3.068.4 2.161 0 3.441-.8 1.308-.8 1.307-2.294 0-1.36-1.147-2.108-1.12-.747-5.068-1.654-5.335-1.253-7.656-3.334-2.32-2.107-2.321-5.362 0-3.76 3.148-6.002 3.148-2.267 7.923-2.267 2.454 0 4.348.507t3.388 1.493q1.493.988 2.454 2.188l.987 1.2-4.482 3.255-.667-.774a7.4 7.4 0 0 0-1.627-1.387 6.4 6.4 0 0 0-2.054-.88 9.5 9.5 0 0 0-2.401-.294q-1.974 0-3.201.8-1.2.774-1.2 1.921 0 1.254 1.12 2.027 1.147.748 5.229 1.681 5.334 1.227 7.629 3.308 2.294 2.055 2.294 5.308 0 4.029-3.228 6.43Q123.217 39 117.989 39M87.61 39q-4.508 0-7.335-2.454-2.828-2.481-2.828-6.482 0-4.188 3.254-6.669t8.483-2.48q2.082 0 4.028.426 1.974.427 3.361 1.12v-1.6q0-2.588-1.813-4.082-1.788-1.493-4.615-1.493-1.628 0-2.801.373a8.7 8.7 0 0 0-2.188.987q-.987.614-1.84 1.414l-.854.8-4.081-3.788 1.307-1.173a18.7 18.7 0 0 1 2.934-2.108q1.601-.933 3.601-1.44t4.589-.507q5.868 0 9.229 2.88 3.361 2.882 3.361 8.084V38.2h-6.749v-5.922l1.654 2.187h-1.76q-1.308 2.055-3.655 3.308Q90.545 39 87.611 39m1.814-4.988q3.148 0 5.149-2.187 2-2.215 2-5.309v-.08q-1.12-.693-2.72-1.094a13 13 0 0 0-3.388-.426q-2.828 0-4.562 1.2-1.734 1.173-1.734 3.521 0 2.028 1.468 3.201t3.787 1.174M49 38.2V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.07V21.794q0-2.827-1.386-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.815 2.027-1.814 4.962v15.391z"></path></svg></a></div></div></div></footer></main><!--$--><!--/$--><script src="/_next/static/chunks/webpack-d716a75e4017380e.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[16174,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"ThemeProvider\"]\n3:I[82180,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"TooltipProvider\"]\n4:I[63851,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"UrqlProvider\"]\n5:I[82714,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"ConsentManager\"]\n6:I[9766,[],\"\"]\n7:I[98924,[],\"\"]\n8:I[33293,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"4345\",\"static/chunks/app/not-found-b861935655c5698b.js\"],\"default\"]\n9:I[21447,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"Integrations\"]\na:I[35514,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"Analytics\"]\nc:I[24431,[],\"OutletBoundary\"]\ne:I[15278,[],\"AsyncMetadataOutlet\"]\n10:I[24431,[],\"ViewportBoundary\"]\n12:I[24431,[],\"MetadataBoundary\"]\n13:\"$Sreact.suspense\"\n15:I[57150,[],\"\"]\n:HL[\"/_next/static/media/304eba2dc288a929-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/38077b822d9512d7-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/40ddcd2db5e33048-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/707228f0d4113e57-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/a73fce2732cf3a48-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/c25783985a269397-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/c86b497a7be35730-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/d9cd7618114896c5-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/f4e6f3e26e87864d-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/fde6a32e2c53c69f-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/css/bc5c0eaecd39bc7d.css\",\"style\"]\n:HL[\"/_next/static/css/7ffcc02a3c67389b.css\",\"style\"]\n:HL[\"/_next/static/css/c20e76e7d11dec32.css\",\"style\"]\n:HL[\"/_next/static/css/a6eb8c1f37be68d5.css\",\"style\"]\n0:{\"P\":null,\"b\":\"ePiaRevyKQ9kV7jt41IDP\",\"p\":\"\",\"c\":[\"\",\"automated-telemetry-capture-via-python-bytecode-modification\"],\"i\":false,\"f\":[[[\"\",{\"children\":[[\"slug\",\"automated-telemetry-capture-via-python-bytecode-modification\",\"c\"],{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],[\"\",[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/bc5c0eaecd39bc7d.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/7ffcc02a3c67389b.css\",\"precedence\":\"next\",\"crossO"])</script><script>self.__next_f.push([1,"rigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"2\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/c20e76e7d11dec32.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"3\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/a6eb8c1f37be68d5.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"suppressHydrationWarning\":true,\"children\":[[\"$\",\"head\",null,{\"children\":[\"$\",\"script\",null,{\"type\":\"text/javascript\",\"dangerouslySetInnerHTML\":{\"__html\":\"window.addEventListener('message',function(e){var d=e.data;if(!d||d.__hnWidget!==1||typeof d.h!=='number'||d.h\u003c=0)return;var f=document.querySelectorAll('.hn-widget-frame');for(var i=0;i\u003cf.length;i++){if(f[i].contentWindow===e.source){f[i].style.height=d.h+'px';break;}}});\"}}]}],[\"$\",\"body\",null,{\"className\":\"__variable_ede2c9 __variable_558dcf bg-background dark:bg-background\",\"children\":[\"$\",\"$L2\",null,{\"attribute\":\"class\",\"defaultTheme\":\"system\",\"enableSystem\":true,\"disableTransitionOnChange\":true,\"children\":[\"$\",\"$L3\",null,{\"children\":[\"$\",\"$L4\",null,{\"children\":[\"$\",\"$L5\",null,{\"children\":[[\"$\",\"$L6\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L7\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"$L8\",null,{}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}],[\"$\",\"$L9\",null,{}],[\"$\",\"$La\",null,{}]]}]}]}]}]}]]}]]}],{\"children\":[[\"slug\",\"automated-telemetry-capture-via-python-bytecode-modification\",\"c\"],[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L6\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L7\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[\"__PAGE__\",[\"$\",\"$1\",\"c\",{\"children\":[\"$Lb\",null,[\"$\",\"$Lc\",null,{\"children\":[\"$Ld\",[\"$\",\"$Le\",null,{\"promise\":\"$@f\"}]]}]]}],{},null,false]},null,false]},null,false],[\"$\",\"$1\",\"h\",{\"children\":[null,[[\"$\",\"$L10\",null,{\"children\":\"$L11\"}],[\"$\",\"meta\",null,{\"name\":\"next-size-adjust\",\"content\":\"\"}]],[\"$\",\"$L12\",null,{\"children\":[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$13\",null,{\"fallback\":null,\"children\":\"$L14\"}]}]}]]}],false]],\"m\":\"$undefined\",\"G\":[\"$15\",[]],\"s\":false,\"S\":false}\n11:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}]]\nd:null\n16:I[80622,[],\"IconMark\"]\nf:{\"metadata\":[[\"$\",\"title\",\"0\",{\"children\":\"Automating Telemetry Capture in Python using Bytecode\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"Discover our journey of automating telemetry capture in Python using bytecode hooks, the challenges we faced, and the alternatives we're considering\"}],[\"$\",\"link\",\"2\",{\"rel\":\"canonical\",\"href\":\"https://jaywhy13.hashnode.dev/automated-telemetry-capture-via-python-bytecode-modification\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:title\",\"content\":\"Automating Telemetry Capture in Python using Bytecode\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:description\",\"content\":\"Discover our journey of automating telemetry capture in Python using bytecode hooks, the challenges we faced, and the alternatives we're considering\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:url\",\"content\":\"https://jaywhy13.hashnode.dev/automated-telemetry-capture-via-python-bytecode-modification\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:site_name\",\"content\":\"Perspective Unspoken\"}],[\"$\",\"meta\",\"7\",{\"property\":\"og:image\",\"content\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg\"}],[\"$\",\"meta\",\"8\",{\"property\":\"og:image:width\",\"content\":\"1200\"}],[\"$\",\"meta\",\"9\",{\"property\":\"og:image:height\",\"content\":\"630\"}],[\"$\",\"meta\",\"10\",{\"property\":\"og:type\",\"content\":\"article\"}],[\"$\",\"meta\",\"11\",{\"property\":\"article:published_time\",\"content\":\"2024-06-25T01:18:44.170Z\"}],[\"$\",\"meta\",\"12\",{\"property\":\"article:author\",\"content\":\"https://jaywhy13.hashnode.dev"])</script><script>self.__next_f.push([1,"/@jaywhy13\"}],[\"$\",\"meta\",\"13\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"14\",{\"name\":\"twitter:creator\",\"content\":\"https://twitter.com/kramnaej\"}],[\"$\",\"meta\",\"15\",{\"name\":\"twitter:title\",\"content\":\"Automating Telemetry Capture in Python using Bytecode\"}],[\"$\",\"meta\",\"16\",{\"name\":\"twitter:description\",\"content\":\"Discover our journey of automating telemetry capture in Python using bytecode hooks, the challenges we faced, and the alternatives we're considering\"}],[\"$\",\"meta\",\"17\",{\"name\":\"twitter:image\",\"content\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg\"}],[\"$\",\"link\",\"18\",{\"rel\":\"icon\",\"href\":\"/favicon.ico\"}],[\"$\",\"$L16\",\"19\",{}]],\"error\":null,\"digest\":\"$undefined\"}\n14:\"$f:metadata\"\n17:I[652,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"8974\",\"static/chunks/app/page-cc18ae1a3a7a5ed8.js\"],\"BlogHeader\"]\n18:T860b,\u003ch2 id=\"heading-introduction\"\u003eIntroduction\u003c/h2\u003e\n\u003cp\u003eEver heard the expression \"Don't \u003cstrong\u003ebyte\u003c/strong\u003e off more than you can chew\"? Today, I've got one of those stories about our attempt to capture telemetry automatically using Python by introducing hooks at the \u003cem\u003ebytecode\u003c/em\u003e level. Our company has several teams managing 20+ microservices and thousands of endpoint. We have a successful manual telemetry capture pattern that significantly reduces incident investigation time by enriching and attaching telemetry to our tools like DataDog, Sentry, and Sumo. However, it's too time-consuming for widespread adoption across all teams. Our team implemented this pattern efficiently, but replicating it manually for other teams would take years! This lead us to explore and eventually \u003cem\u003eabandon\u003c/em\u003e an overly complex automated bytecode capture approach.\u003c/p\u003e\n\u003cp\u003eBelow is the story of that journey.\u003c/p\u003e\n\u003ch2 id=\"heading-motivation\"\u003eMotivation\u003c/h2\u003e\n\u003cp\u003eUltimately we want to increase our business uptime by reducing incident recovery time. We achieve this by enhancing our production experience through access to high quality telemetry for investigation. We're aiming to replicate a local debugger in production. Enhancing system state visibility accelerates our investigations, which reduces disruption time. If we're able to clearly see system state when looking at a trace, or an error, or log, we'll drastically reduce remediation time. High quality telemetry provides a solid base for hypotheses creation and testing.\u003c/p\u003e\n\u003cp\u003eWe've had successes that confirm that this approach works too. We had an incident where we inadvertently gave \u003ca target=\"_blank\" href=\"https://jaywhy13.hashnode.dev/solving-like-sherlock-a-15-minute-case-with-observability\"\u003efree access\u003c/a\u003e to some premium features. We got to our root cause for that issue in less than 15 minutes thanks to the tooling improvements. In that incident, we zoomed out from an individual trace and aggregated over a suspicious property which confirmed our theory. We had another incident where we fixed a broken Kafka worker and it caused data corruption for a running migration. Again, examining traces and inspecting system state was critical for understanding issues quickly.\u003c/p\u003e\n\u003ch2 id=\"heading-step-0-the-existing-manual-process\"\u003eStep 0 - The existing, manual process\u003c/h2\u003e\n\u003cp\u003eLet's take a look at what our manual process looks like. Below I'm showing the code for a \u003ccode\u003eSubscriptionAPI\u003c/code\u003e. This represents the highest component in our architecture. An API typically calls a service, then a repository for data operations. You'll see there's a \u003ccode\u003eContext\u003c/code\u003e (alias for a Python dictionary) that we start adding information to. That's us collecting telemetry. Below we're adding the input values to the \u003ccode\u003eContext\u003c/code\u003e like the \u003ccode\u003eapi\u003c/code\u003e, \u003ccode\u003eus"])</script><script>self.__next_f.push([1,"er_id\u003c/code\u003e, \u003ccode\u003ebusiness_id\u003c/code\u003e, \u003ccode\u003eplan_id\u003c/code\u003e (the subscription's plan) and eventually the \u003ccode\u003eserialized_subscription\u003c/code\u003e we generate as the API output. The \u003ccode\u003eContext\u003c/code\u003e is then passed down the hierarchy to all other calls. In this scenario, the \u003ccode\u003eSubscriptionAPI\u003c/code\u003e calls the \u003ccode\u003eSubscriptionService.create_subscription\u003c/code\u003e method. It passes the \u003ccode\u003ectx\u003c/code\u003e to that method so we can continue to populate it.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionAPI\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Start capturing telemetry (using a dictionary \"ctx\")\u003c/span\u003e\n ctx[\u003cspan class=\"hljs-string\"\u003e\"api\"\u003c/span\u003e] = \u003cspan class=\"hljs-string\"\u003e\"create_subscription\"\u003c/span\u003e\n ctx[\u003cspan class=\"hljs-string\"\u003e\"user_id\"\u003c/span\u003e] = str(user_id)\n ctx[\u003cspan class=\"hljs-string\"\u003e\"plan_id\"\u003c/span\u003e] = str(plan_id)\n ctx[\u003cspan class=\"hljs-string\"\u003e\"business_id\"\u003c/span\u003e] = str(business_id)\n\n \u003cspan class=\"hljs-comment\"\u003e# Call the service to create a subscription\u003c/span\u003e\n subscription = SubscriptionService().create_subscription(\n ctx, plan_id=plan_id, user_id=user_id, business_id=business_id\n )\n \u003cspan class=\"hljs-comment\"\u003e# Serialize the subscription to JSON for return\u003c/span\u003e\n serialized_subscription = {\n \u003cspan class=\"hljs-string\"\u003e\"id\"\u003c/span\u003e: str(subscription.id),\n \u003cspan class=\"hljs-string\"\u003e\"plan_id\"\u003c/span\u003e: str(subscription.plan_id),\n \u003cspan class=\"hljs-string\"\u003e\"user_id\"\u003c/span\u003e: str(subscription.user_id),\n \u003cspan class=\"hljs-string\"\u003e\"business_id\"\u003c/span\u003e: str(subscription.business_id),\n \u003cspan class=\"hljs-string\"\u003e\"status\"\u003c/span\u003e: subscription.status.value,\n }\n ctx[\u003cspan class=\"hljs-string\"\u003e\"serialized_subscription\"\u003c/span\u003e] = serialized_subscription\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e serialized_subscription\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eAs we populate the \u003ccode\u003ectx\u003c/code\u003e in each layer of the application, we end up with a rich description of our current system state.\u003c/p\u003e\n\u003cp\u003eBelow is an example of us invoke the \u003ccode\u003eSubscriptionAPI\u003c/code\u003e and then examining the \u003ccode\u003eContext\u003c/code\u003e.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-comment\"\u003e# Declare variables (plan_id, user_id, etc...)\u003c/span\u003e\nctx = {}\n\nsubscription_api = SubscriptionAPI()\nsubscription_api.create_subscription(\n ctx, plan_id, user_id, business_id)\n\nprint(ctx)\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThe output of the \u003ccode\u003eContext\u003c/code\u003e looks like this:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-json\"\u003e{\u003cspan class=\"hljs-attr\"\u003e\"api\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"create_subscription\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"business_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"bc7464cc-dd17-481e-88a4-93ba5711f2a1\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"plan_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"43a3ec11-ba1d-4fff-a958-f3945fc46a36\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"serialized_subscription\"\u003c/span\u003e: {\u003cspan class=\"hljs-attr\"\u003e\"business_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"bc7464cc-dd17-481e-88a4-93ba5711f2a1\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"f2dc517e-5c9c-4f55-8a26-ef43dd7bc0ef\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"plan_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"43a3ec11-ba1d-4fff-a958-f3945fc46a36\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"status\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"active\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"user_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"61346915-3c35-4ba4-87fd-7d092053c940\"\u003c/span\u003e},\n \u003cspan class=\"hljs-attr\"\u003e\"status\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"active\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"user_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"61346915-3c35-4ba4-8"])</script><script>self.__next_f.push([1,"7fd-7d092053c940\"\u003c/span\u003e}\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThis manual capture approach works well when you're starting fresh and you establish this as a team practice. However, when you've got thousands of endpoints scattered across multiple services without the pattern, we need an automated approach.\u003c/p\u003e\n\u003cp\u003eLet's dive into the steps we took on the journey to automated telemetry capture.\u003c/p\u003e\n\u003ch2 id=\"heading-step-1-capture-variables-in-a-method\"\u003eStep 1 - Capture variables in a method\u003c/h2\u003e\n\u003cp\u003eThe first step is understanding how we capture all the variables in a method. Once we can capture variables in one method, we can extend this approach to the rest of the architecture stack. Python provides a facility for accomplishing this. We can use \u003ccode\u003einspect.stack()\u003c/code\u003e to get local variables from a function. The stack stores variables for each function being called. Python creates a frame for each function being called. Let's see how this works in practice.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionAPI\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Call the service to create a subscription\u003c/span\u003e\n subscription = SubscriptionService().create_subscription(\n ctx, plan_id=plan_id, user_id=user_id, business_id=business_id\n )\n \u003cspan class=\"hljs-comment\"\u003e# Serialize the subscription to JSON for return\u003c/span\u003e\n serialized_subscription = {...}\n\n \u003cspan class=\"hljs-comment\"\u003e# Capture all the variables in the method\u003c/span\u003e\n stack = inspect.stack()\n local_variables = stack[\u003cspan class=\"hljs-number\"\u003e0\u003c/span\u003e].frame.f_locals\n\n \u003cspan class=\"hljs-comment\"\u003e# Capture all the local variables in the context\u003c/span\u003e\n ctx.update(local_variables)\n\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e serialized_subscription\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eNear the end of our \u003ccode\u003ecreate_subscription\u003c/code\u003e function, we're accessing the \u003ccode\u003estack\u003c/code\u003e and printing the locals in the first frame (i.e. at index \u003ccode\u003e0\u003c/code\u003e). This gives us all the local variables in the current function (i.e. \u003ccode\u003ecreate_subscription\u003c/code\u003e). Every time Python calls a function, it creates a frame for it and stores the locals.\u003c/p\u003e\n\u003cp\u003eIf we run the \u003ccode\u003ecreate_subscription\u003c/code\u003e method and examine the \u003ccode\u003ectx\u003c/code\u003e, it looks like this:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e{\u003cspan class=\"hljs-string\"\u003e'business_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'bbe18a98-d565-4423-a13a-1c17b38b003d'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'plan_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'81df53ad-2159-4566-b43f-3c24d55d9d3e'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'serialized_subscription'\u003c/span\u003e: {\u003cspan class=\"hljs-string\"\u003e'business_id'\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e'bbe18a98-d565-4423-a13a-1c17b38b003d'\u003c/span\u003e,\n \u003cspan class=\"hljs-string\"\u003e'id'\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e'977ac956-496f-4b54-8627-a0fd4fc3e9df'\u003c/span\u003e,\n \u003cspan class=\"hljs-string\"\u003e'plan_id'\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e'81df53ad-2159-4566-b43f-3c24d55d9d3e'\u003c/span\u003e,\n \u003cspan class=\"hljs-string\"\u003e'status'\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e'active'\u003c/span\u003e,\n \u003cspan class=\"hljs-string\"\u003e'user_id'\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e'ade39122-70b9-47f8-94e9-6ab4926e7398'\u003c/span\u003e},\n \u003cspan class=\"hljs-string\"\u003e'subscription'\u003c/span\u003e: Subscription(id=UUID(\u003cspan class=\"hljs-string\"\u003e'977ac956-496f-4b54-8627-a0fd4fc3e9df'\u003c/span\u003e),\n plan_id=UUID(\u003cspan class=\"hljs-string\"\u003e'81df53ad-2159-4566-b43f-3c24d55d9d3e'\u003c/span\u003e),\n user_id=UUID(\u003cspan class=\"hljs-string\"\u003e'ade39122-70b9-47f8-94e9-6ab4926e7398'\u003c/span\u003e),\n business_id=UUID(\u003cspan class=\"hljs-string\"\u003e'b"])</script><script>self.__next_f.push([1,"be18a98-d565-4423-a13a-1c17b38b003d'\u003c/span\u003e),\n status=\u0026lt;SubscriptionStatus.ACTIVE: \u003cspan class=\"hljs-string\"\u003e'active'\u003c/span\u003e\u0026gt;),\n \u003cspan class=\"hljs-string\"\u003e'user_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'ade39122-70b9-47f8-94e9-6ab4926e7398'\u003c/span\u003e)}\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eAs you see from the output, we've captured all the fields from the \u003ccode\u003ecreate_subscription\u003c/code\u003e method.\u003c/p\u003e\n\u003cp\u003eLet's add a visual to concretize the concept.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1718937071544/663c1e0b-b193-4233-b1d7-53e58492e5b1.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eWhen we call \u003ccode\u003eSubscriptionAPI.create_subscription\u003c/code\u003e, Python pushes a frame unto the stack. That frame contains the locals of our method.\u003c/p\u003e\n\u003cp\u003eSounds good! We're making progress.\u003c/p\u003e\n\u003ch2 id=\"heading-step-2-capture-the-locals-from-functions-we-call\"\u003eStep 2 - Capture the locals from functions we call\u003c/h2\u003e\n\u003cp\u003eWe don't only need to capture the locals from the \u003ccode\u003eSubscriptionAPI\u003c/code\u003e call, but all the other components (e.g. services, repositories, clients) it calls to satisfy our request. The \u003ccode\u003eSubscriptionAPI\u003c/code\u003e will call the \u003ccode\u003eSubscriptionService\u003c/code\u003e to create the subscription. The \u003ccode\u003eSubscriptionService\u003c/code\u003e will in turn, call the repository (\u003ccode\u003eSubscriptionRepository\u003c/code\u003e) to convince the database to create the data.\u003c/p\u003e\n\u003cp\u003eLet's look at a quick sequence diagram to concretize the picture in our minds.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1718937505858/36b51f30-f97b-4445-a337-6e65e921caf2.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eThankfully, Python can help us here too! Python captures all the variables for all the functions that get called. We learned earlier that each time Python invokes a function, it creates a stack frame with the locals. When a function calls another function, Python creates a new stack frame for the new function and its locals. So as functions are calling other functions, Python pushes new frames unto the stack. As such, we just need to loop through each frame to capture all the variables.\u003c/p\u003e\n\u003cp\u003eLet's look at the code for each of the components. They're mostly passthroughs.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionRepository\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self,\n ctx: Context,\n plan_id: UUID,\n user_id: UUID,\n business_id: UUID,\n status: SubscriptionStatus,\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Pretend we're calling the ORM to create the subscription\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Serialize the output as a Subcription value object\u003c/span\u003e\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e Subscription(...)\n\n\n\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionService\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n \u003c/span\u003e) -\u0026gt; Subscription:\u003c/span\u003e\n status = SubscriptionStatus.ACTIVE\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e SubscriptionRepository().create_subscription(...)\n\n\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionAPI\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Call the service to create a subscription\u003c/span\u003e\n subscription = SubscriptionService().create_subscription(...)\n \u003cspan class=\"hljs-comment\"\u003e# Serialize the subscript"])</script><script>self.__next_f.push([1,"ion to JSON for return\u003c/span\u003e\n serialized_subscription = {...}\n\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e serialized_subscription\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eIf we were to look at the stack, when the \u003ccode\u003eSubscriptionRepository.create_subscription\u003c/code\u003e function is being interpreted, the stack would contain frames for the calling components (i.e. API and service).\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1718938824076/1699c24c-4493-4a4d-9d7f-98408a549038.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eFirst, there's a frame for the current method (i.e. \u003ccode\u003eSubscriptionRepository.create_subscription\u003c/code\u003e) with all its variables. It was called by \u003ccode\u003eSubscriptionService.create_subscription\u003c/code\u003e, so there's a stack for that. Finally, the service was called by \u003ccode\u003eSubscriptionAPI.create_subscription\u003c/code\u003e so there's a frame for that one as well.\u003c/p\u003e\n\u003cp\u003eLet's look at how we'd achieve that.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eget_variables_from_all_calls\u003c/span\u003e() -\u0026gt; dict:\u003c/span\u003e\n \u003cspan class=\"hljs-string\"\u003e\"\"\"Get all the variables in the method that called us.\"\"\"\u003c/span\u003e\n local_variables = {}\n stack = inspect.stack()\n \u003cspan class=\"hljs-comment\"\u003e# We don't want the current frame, that's this function\u003c/span\u003e\n all_frames_except_this_one = stack[\u003cspan class=\"hljs-number\"\u003e1\u003c/span\u003e:]\n \u003cspan class=\"hljs-keyword\"\u003efor\u003c/span\u003e frame_information \u003cspan class=\"hljs-keyword\"\u003ein\u003c/span\u003e all_frames_except_this_one:\n frame = frame_information.frame\n local_variables.update(frame.f_locals)\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e local_variables\n\n\n\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionRepository\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self,\n ctx: Context,\n plan_id: UUID,\n user_id: UUID,\n business_id: UUID,\n status: SubscriptionStatus,\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Call the ORM to create the subscription\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Update the context with variables from all calls in the stack\u003c/span\u003e\n ctx.update(get_variables_from_all_calls()) \n \u003cspan class=\"hljs-comment\"\u003e# Serialize the output as a Subcription object\u003c/span\u003e\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e Subscription(...)\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eIn this example, I'm introducing a helper (\u003ccode\u003eget_variables_from_all_calls\u003c/code\u003e) to move the logic outside of our main function. Since the repository will call this method, that means Python will add a frame to the stack for this helper function as well. As such, we need to get all the frames except the current one. This explains why we're only capturing frames from index \u003ccode\u003e1\u003c/code\u003e and beyond. Finally, in \u003ccode\u003eSubscriptionRepository.create_subscription\u003c/code\u003e we're calling \u003ccode\u003eget_variables_from_all_calls\u003c/code\u003e to get all the variables for us.\u003c/p\u003e\n\u003cp\u003eWhen we run the code, the \u003ccode\u003ectx\u003c/code\u003e has the following contents:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e{\u003cspan class=\"hljs-string\"\u003e'business_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'3498a3df-5001-481c-b0b8-a007609fb93c'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'plan_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'fefbbafd-0f59-49bb-9255-3c09eef38fec'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'status'\u003c/span\u003e: \u0026lt;SubscriptionStatus.ACTIVE: \u003cspan class=\"hljs-string\"\u003e'active'\u003c/span\u003e\u0026gt;,\n \u003cspan class=\"hljs-string\"\u003e'user_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'de19ca36-67c5-48eb-80c8-70b0db9575ac'\u003c/span\u003e)}\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eIf you're paying close attention, you'll notice this list is a little shorter than the last time we printed out the \u003ccode\u003eContext\u003c/code\u003e. In particular, this is missing the \u003ccode\u003eserialized_subscription\u003c/code\u003e and \u003ccode\u003esubscription\u003c/code\u003e object found in the \u003ccode\u003eSubscriptionAPI.create_subscription\u003c/code\u003e method. That happ"])</script><script>self.__next_f.push([1,"ens because when we're capturing the variables, those don't exist yet.\u003c/p\u003e\n\u003cp\u003eSee below if you're not convinced.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionAPI\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# The subscription variable is defined AFTER the call returns\u003c/span\u003e\n subscription = SubscriptionService().create_subscription(...)\n \u003cspan class=\"hljs-comment\"\u003e# Similarly, this is also defined AFTER the call returns\u003c/span\u003e\n serialized_subscription = {...}\n\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e serialized_subscription\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eWe'll keep going! That's a small problem in the grand scheme of things. As you'll see in the next step... we've got bigger problems.\u003c/p\u003e\n\u003ch2 id=\"heading-step-3-dont-touch-the-code\"\u003eStep 3 - Don't touch the code\u003c/h2\u003e\n\u003cp\u003eOne of our primary goals was to enable telemetry without manual effort. If we need to be adding a bunch of \u003ccode\u003eget_variables_from_all_calls\u003c/code\u003e calls all over our code, that simply won't work. It's less work than manually adding every variable, but still too much effort. Additionally, we don't want folks to modify their code to include these calls. We want these utilities to be mostly invisible but providing all the benefits necessary.\u003c/p\u003e\n\u003cp\u003eLet's iterate and see what we can accomplish...\u003c/p\u003e\n\u003cp\u003eWhat if we tried capturing the stack variables after the we made the function call.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003esubscription_api = SubscriptionAPI()\nctx = {}\n\nsubscription_api.create_subscription(\n ctx, plan_id, user_id, business_id)\n\n\u003cspan class=\"hljs-comment\"\u003e# Capture all variables\u003c/span\u003e\nget_variables_from_all_calls()\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eIf we print out the context of the \u003ccode\u003eContext\u003c/code\u003e, we'd get the following:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e{}\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThe \u003ccode\u003eContext\u003c/code\u003e is empty. Yeah, that's right... it's completely empty.\u003c/p\u003e\n\u003cp\u003eHere comes another important detail I omitted before...\u003c/p\u003e\n\u003cblockquote\u003e\n\u003cp\u003e\u003cstrong\u003eOnce a function is done executing, Python pops its frame from the stack.\u003c/strong\u003e\u003c/p\u003e\n\u003c/blockquote\u003e\n\u003cp\u003eSo... if you were to look at the stack at this point, it'd be empty (technically there's a \"module\" frame, but I've omitted that for brevity). There are no other frames on the stack, because after the function calls have exited, the frames associated with each call are completely gone. All of the previous approaches worked because we were capturing variables while we were \u003cem\u003ein\u003c/em\u003e the call stack. Once those functions have exited we're no longer in the call stack and don't have access to those variables.\u003c/p\u003e\n\u003cp\u003eSo we need a different approach. We need to capture variables while the relevant frames still exist on the stack. But how...\u003c/p\u003e\n\u003ch2 id=\"heading-step-4-byte-off-more-than-you-can-chew\"\u003eStep 4 - Byte off more than you can chew\u003c/h2\u003e\n\u003cp\u003eIt turns out, there's a way we can utilize the same approach, without modifying the code folks would write. We can introduce our capturing logic, at a lower-level... much lower. When we write Python code, it gets compiled into bytecode, which is then run by the interpreter.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1719277675826/ba7ff392-856d-4314-b83b-2cfa8ad75c03.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eWe could \u003cem\u003emodify\u003c/em\u003e the bytecode to accomplish the same objective. Python bytecode looks very different from Python. It's a sequence of instructions for the Python virtual machine. These instructions manipulate the internal stacks that Python uses. By modifying the resulting bytecode we can inject our logic and have the compiler execute our logic and capture the variables.\u003c/p\u003e\n\u003cp\u003eTo exactly replicate what we did at a higher level, we need to capture the stack just before the functio"])</script><script>self.__next_f.push([1,"n returns. So, we'd need to parse the bytecode instructions, looking for a return statement and then inject our logic right before that. We'd insert our logic to call the \u003ccode\u003eget_variables_from_all_calls\u003c/code\u003e function which would extract the variables from the stack. Then finally we'd relace the function's bytecode with our patched version. Sounds simple enough, right?\u003c/p\u003e\n\u003cp\u003eThere are lots of different bytecode instructions. The following are the ones we're interested in:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003cp\u003e\u003ccode\u003eLOAD_GLOBAL\u003c/code\u003e - this instruction enables us to make our function available for calling. This instruction tells Python to look up an object in the global namespace and place it on the evaluation stack. We'll use this to make our function ready for calling.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003ccode\u003eLOAD_FAST\u003c/code\u003e - this instruction enables us to provide arguments for a function call. This pushes a variable unto the stack for evaluation.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003ccode\u003eCALL_FUNCTION\u003c/code\u003e - this instruction is used to execute our function call. With this instruction we can also provide a number of positional arguments that should be popped and passed to the function.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003ccode\u003ePOP_TOP\u003c/code\u003e - this instruction allows us to remove our function from the stack to continue the flow of the program. We'll use this to pop our function after we've called it.\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003eNext, I need to explain an idiosyncrasy we have to be familiar with to make this all work. Sometimes instructions spans multiple lines. We only need to add our patching instructions once. As such, we need to keep track of whether we've patched a line already so we don't do it multiple times.\u003c/p\u003e\n\u003cp\u003eArmed with that knowledge let's examine the function that does the patching.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003epatch_function_to_capture_stack_variables\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003efunc\u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Get the bytecode that we plan to patch\u003c/span\u003e\n code = func.__code__\n bytecode = Bytecode.from_code(code)\n\n \u003cspan class=\"hljs-comment\"\u003e# Get a list of all line numbers for return instructions\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Recall instructions can span multiple lines\u003c/span\u003e\n return_line_numbers = [\n instruction.lineno\n \u003cspan class=\"hljs-keyword\"\u003efor\u003c/span\u003e instruction \u003cspan class=\"hljs-keyword\"\u003ein\u003c/span\u003e bytecode\n \u003cspan class=\"hljs-keyword\"\u003eif\u003c/span\u003e instruction.name == \u003cspan class=\"hljs-string\"\u003e\"RETURN_VALUE\"\u003c/span\u003e\n ]\n\n \u003cspan class=\"hljs-comment\"\u003e# For each return instruction, we want to insert a call to capture_stack\u003c/span\u003e\n \u003cspan class=\"hljs-keyword\"\u003efor\u003c/span\u003e return_line_number \u003cspan class=\"hljs-keyword\"\u003ein\u003c/span\u003e return_line_numbers:\n \u003cspan class=\"hljs-comment\"\u003e# construct the instructions we need\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Add our function to the stack\u003c/span\u003e\n add_capture_stack_function_to_the_stack = Instr(\u003cspan class=\"hljs-string\"\u003e\"LOAD_GLOBAL\"\u003c/span\u003e, \u003cspan class=\"hljs-string\"\u003e\"get_variables_from_all_calls\"\u003c/span\u003e)\n \u003cspan class=\"hljs-comment\"\u003e# Add the ctx as an argument for the function\u003c/span\u003e\n add_ctx_as_an_argument = Instr(\u003cspan class=\"hljs-string\"\u003e\"LOAD_FAST\"\u003c/span\u003e, \u003cspan class=\"hljs-string\"\u003e\"ctx\"\u003c/span\u003e)\n \u003cspan class=\"hljs-comment\"\u003e# Call our function and pop 1 arguemnt (ctx) to be passed to the function\u003c/span\u003e\n call_the_capture_stack_function = Instr(\u003cspan class=\"hljs-string\"\u003e\"CALL_FUNCTION\"\u003c/span\u003e, \u003cspan class=\"hljs-number\"\u003e1\u003c/span\u003e)\n \u003cspan class=\"hljs-comment\"\u003e# Remove our function to continue normal processing\u003c/span\u003e\n remove_the_capture_stack_function_from_the_stack = Instr(\u003cspan class=\"hljs-string\"\u003e\"POP_TOP\"\u003c/span\u003e)\n \u003cspan class=\"hljs-comment\"\u003e# Put the instructions all together\u003c/span\u003e\n catpure_stack_instructions = [\n add_capture_stack_function_to_the_stack,\n add_ctx_as_an_argument,\n call_the_capture_stack_function,\n remove_the_capture_stack_function_fro"])</script><script>self.__next_f.push([1,"m_the_stack,\n ]\n\n \u003cspan class=\"hljs-keyword\"\u003efor\u003c/span\u003e i, instruction \u003cspan class=\"hljs-keyword\"\u003ein\u003c/span\u003e enumerate(bytecode):\n \u003cspan class=\"hljs-keyword\"\u003eif\u003c/span\u003e instruction.lineno == return_line_number:\n bytecode[i:i] = catpure_stack_instructions\n \u003cspan class=\"hljs-comment\"\u003e# There could be other instructions on the same line\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# We don't want to insert the new instructions multiple times\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# So break out of the loop\u003c/span\u003e\n \u003cspan class=\"hljs-keyword\"\u003ebreak\u003c/span\u003e\n\n updated_code = bytecode.to_code()\n\n func.__code__ = updated_code\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eAs shown in the code above, we're getting the bytecode, adding instructions to call our function and then applying the patch. We start by getting the \u003ccode\u003e__code__\u003c/code\u003e object of the function. That's the bytecode! We're using the \u003ccode\u003ebytecode\u003c/code\u003e library to simplify going to and from bytecode and construction of instructions. Next, we find all the lines that have return instructions. Finally, we construct our new instructions and add them to the bytecode.\u003c/p\u003e\n\u003cp\u003eFinally, we patch the \u003ccode\u003eSubscriptionRepository.create_subscription\u003c/code\u003e function so it'll call our function.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003epatch_function_to_capture_stack_variables(\n SubscriptionRepository.create_subscription)\n\nsubscription_api = SubscriptionAPI()\nsubscription_api.create_subscription(\n ctx, plan_id, user_id, business_id)\n\nprint(ctx)\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eRunning this code, gets us the following output:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e {\u003cspan class=\"hljs-string\"\u003e'business_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'276dac6c-68ad-48bf-8a55-ac7ee6df64a5'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'plan_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'e72b5c80-63d7-4c5a-a2a9-d4b5eef937e0'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'status'\u003c/span\u003e: \u0026lt;SubscriptionStatus.ACTIVE: \u003cspan class=\"hljs-string\"\u003e'active'\u003c/span\u003e\u0026gt;,\n \u003cspan class=\"hljs-string\"\u003e'user_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'946bf774-e95a-488d-b5b5-55a255aa5d2a'\u003c/span\u003e)}\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eSuccess! Without asking anyone to modify their code, we were able to capture the local variables from the stack! This provides significant gains as we're able to reduce the manual effort involved. It's leaps and bounds further than where we started.\u003c/p\u003e\n\u003cp\u003eYou'll notice though that it's the same output as step 3. It still has the shortcoming where it doesn't capture variables that haven't been defined yet. There are ways to overcome this that don't take too much effort and don't re-introduce manual effort. Anyway... that's a story for another time.\u003c/p\u003e\n\u003ch2 id=\"heading-step-5-choke\"\u003eStep 5 - Choke\u003c/h2\u003e\n\u003cp\u003eI've had to acknowledge that despite the energy rush this investigation brought, its complexity is well beyond our company's maintenance capacity. None of us are experts in the topic of bytecode. Theoretically, we're not doing much -- we're simply hooking into the code at a lower level. We wanted to avoid folks littering their code with our hooking logic, so we moved down an abstraction layer or two. That descent was unfortunately dangerously close to 6 feet under. Writing our Observability tooling hooks in bytecode would be suicidal for our tooling's life.\u003c/p\u003e\n\u003cp\u003eWhat's worse, is that the implementation I've described thus far is woefully inadequate. We'd need to make several improvements and additions to \u003cem\u003ebegin\u003c/em\u003e the journey to production. We'd need some more bytecode instructions to import our library from elsewhere (in the script it's defined in the same file). Discover and account for additional idiosyncrasies of bytecode. We haven't addressed the multiple ways that functions can exit (e.g. early returns, unexpected exceptions and try-catch scenarios). Additionally, we would want to make our code more robust. We don't want failures in our library to cascade to the rest of the application. We'd probably need to catch all failures in our calls and properly log"])</script><script>self.__next_f.push([1," them (e.g. Sentry, SumoLogic, DataDog, etc...), or capture with metrics. Then, finally, we haven't thought about optimization or security. Who knows what challenges lie in that space!\u003c/p\u003e\n\u003cp\u003eAnd so, we've started investigating alternatives. We've got some basic ideas we're toying around. One idea is to reduce the complexity, by only capturing function inputs and outputs. We could use a decorator-based approach to wrap public methods of all our components and capture the variables. We can implement our wrapping logic in a metaclass that we use to initialize our different components (e.g. services, repositories, clients, etc...). It's early days and we're still exploring things. I'll return to blog about our success (let's hope!) once we've found a working solution.\u003c/p\u003e\n\u003ch2 id=\"heading-conclusion\"\u003eConclusion\u003c/h2\u003e\n\u003cp\u003eIn our attempt to automate telemetry capture using Python, we explored the ambitious idea of introducing hooks at the bytecode level to automate this process. While our manual approach of enriching telemetry within our services has proven effective in reducing incident investigation time, scaling this across multiple teams and microservices was impractical. Our journey involved understanding Python's stack and bytecode intricacies to capture local variables across function calls. Despite the surge and adrenaline of this approach, the unavoidable complexity and maintenance challenges, lead us to abandon this method. We're now considering simpler, more maintainable alternatives like using decorators to capture function inputs and outputs.\u003c/p\u003e\n\u003cp\u003eThanks for reading! If you've got thoughts on this topic, I'd love to hear it! Feel free to reach out to me on \u003ca target=\"_blank\" href=\"https://x.com/kramnaej\"\u003eTwitter\u003c/a\u003e or \u003ca target=\"_blank\" href=\"https://www.linkedin.com/in/jean-mark-wright/\"\u003eLinkedIn\u003c/a\u003e!\u003c/p\u003e\n\u003cp\u003eAlso, many thanks to the \u003ca target=\"_blank\" href=\"https://github.com/DataDog/datadogpy\"\u003eDataDog library\u003c/a\u003e I reverse-engineered to get an idea of how this could be accomplished.\u003c/p\u003e\n\u003cp\u003eHere are a few resources that were helpful, if you're looking to learn more about bytecode.\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003cp\u003e\u003ca target=\"_blank\" href=\"https://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1\"\u003eDemystifying Python Bytecode: A Guide to Understanding and Analyzing Code Execution\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003ca target=\"_blank\" href=\"https://github.com/DataDog/datadogpy\"\u003eDataDog Python Library\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003e\u003ca target=\"_blank\" href=\"https://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1\"\u003ehttps://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1\u003c/a\u003e\u003c/p\u003e\n19:T64a8,## Introduction\n\nEver heard the expression \"Don't **byte** off more than you can chew\"? Today, I've got one of those stories about our attempt to capture telemetry automatically using Python by introducing hooks at the *bytecode* level. Our company has several teams managing 20+ microservices and thousands of endpoint. We have a successful manual telemetry capture pattern that significantly reduces incident investigation time by enriching and attaching telemetry to our tools like DataDog, Sentry, and Sumo. However, it's too time-consuming for widespread adoption across all teams. Our team implemented this pattern efficiently, but replicating it manually for other teams would take years! This lead us to explore and eventually *abandon* an overly complex automated bytecode capture approach.\n\nBelow is the story of that journey.\n\n## Motivation\n\nUltimately we want to increase our business uptime by reducing incident recovery time. We achieve this by enhancing our production experience through access to high quality telemetry for investigation. We're aiming to replicate a local debugger in production. Enhancing system state visibility accelerates our investigations, which reduces disruption time. If we're able to clearly see system state when looking at a trace, or "])</script><script>self.__next_f.push([1,"an error, or log, we'll drastically reduce remediation time. High quality telemetry provides a solid base for hypotheses creation and testing.\n\nWe've had successes that confirm that this approach works too. We had an incident where we inadvertently gave [free access](https://jaywhy13.hashnode.dev/solving-like-sherlock-a-15-minute-case-with-observability) to some premium features. We got to our root cause for that issue in less than 15 minutes thanks to the tooling improvements. In that incident, we zoomed out from an individual trace and aggregated over a suspicious property which confirmed our theory. We had another incident where we fixed a broken Kafka worker and it caused data corruption for a running migration. Again, examining traces and inspecting system state was critical for understanding issues quickly.\n\n## Step 0 - The existing, manual process\n\nLet's take a look at what our manual process looks like. Below I'm showing the code for a `SubscriptionAPI`. This represents the highest component in our architecture. An API typically calls a service, then a repository for data operations. You'll see there's a `Context` (alias for a Python dictionary) that we start adding information to. That's us collecting telemetry. Below we're adding the input values to the `Context` like the `api`, `user_id`, `business_id`, `plan_id` (the subscription's plan) and eventually the `serialized_subscription` we generate as the API output. The `Context` is then passed down the hierarchy to all other calls. In this scenario, the `SubscriptionAPI` calls the `SubscriptionService.create_subscription` method. It passes the `ctx` to that method so we can continue to populate it.\n\n```python\nclass SubscriptionAPI:\n\n def create_subscription(\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n ):\n # Start capturing telemetry (using a dictionary \"ctx\")\n ctx[\"api\"] = \"create_subscription\"\n ctx[\"user_id\"] = str(user_id)\n ctx[\"plan_id\"] = str(plan_id)\n ctx[\"business_id\"] = str(business_id)\n\n # Call the service to create a subscription\n subscription = SubscriptionService().create_subscription(\n ctx, plan_id=plan_id, user_id=user_id, business_id=business_id\n )\n # Serialize the subscription to JSON for return\n serialized_subscription = {\n \"id\": str(subscription.id),\n \"plan_id\": str(subscription.plan_id),\n \"user_id\": str(subscription.user_id),\n \"business_id\": str(subscription.business_id),\n \"status\": subscription.status.value,\n }\n ctx[\"serialized_subscription\"] = serialized_subscription\n return serialized_subscription\n```\n\nAs we populate the `ctx` in each layer of the application, we end up with a rich description of our current system state.\n\nBelow is an example of us invoke the `SubscriptionAPI` and then examining the `Context`.\n\n```python\n# Declare variables (plan_id, user_id, etc...)\nctx = {}\n\nsubscription_api = SubscriptionAPI()\nsubscription_api.create_subscription(\n ctx, plan_id, user_id, business_id)\n\nprint(ctx)\n```\n\nThe output of the `Context` looks like this:\n\n```json\n{\"api\": \"create_subscription\",\n \"business_id\": \"bc7464cc-dd17-481e-88a4-93ba5711f2a1\",\n \"plan_id\": \"43a3ec11-ba1d-4fff-a958-f3945fc46a36\",\n \"serialized_subscription\": {\"business_id\": \"bc7464cc-dd17-481e-88a4-93ba5711f2a1\",\n \"id\": \"f2dc517e-5c9c-4f55-8a26-ef43dd7bc0ef\",\n \"plan_id\": \"43a3ec11-ba1d-4fff-a958-f3945fc46a36\",\n \"status\": \"active\",\n \"user_id\": \"61346915-3c35-4ba4-87fd-7d092053c940\"},\n \"status\": \"active\",\n \"user_id\": \"61346915-3c35-4ba4-87fd-7d092053c940\"}\n```\n\nThis manual capture approach works well when you're starting fresh and you establish this as a team practice. However, when you've got thousands of endpoints scattered across multiple services without the pattern, we need an automated approach.\n\nLet's dive into the steps we took on the journey to automated telemetry cap"])</script><script>self.__next_f.push([1,"ture.\n\n## Step 1 - Capture variables in a method\n\nThe first step is understanding how we capture all the variables in a method. Once we can capture variables in one method, we can extend this approach to the rest of the architecture stack. Python provides a facility for accomplishing this. We can use `inspect.stack()` to get local variables from a function. The stack stores variables for each function being called. Python creates a frame for each function being called. Let's see how this works in practice.\n\n```python\nclass SubscriptionAPI:\n\n def create_subscription(\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n ):\n # Call the service to create a subscription\n subscription = SubscriptionService().create_subscription(\n ctx, plan_id=plan_id, user_id=user_id, business_id=business_id\n )\n # Serialize the subscription to JSON for return\n serialized_subscription = {...}\n\n # Capture all the variables in the method\n stack = inspect.stack()\n local_variables = stack[0].frame.f_locals\n\n # Capture all the local variables in the context\n ctx.update(local_variables)\n\n return serialized_subscription\n```\n\nNear the end of our `create_subscription` function, we're accessing the `stack` and printing the locals in the first frame (i.e. at index `0`). This gives us all the local variables in the current function (i.e. `create_subscription`). Every time Python calls a function, it creates a frame for it and stores the locals.\n\nIf we run the `create_subscription` method and examine the `ctx`, it looks like this:\n\n```python\n{'business_id': UUID('bbe18a98-d565-4423-a13a-1c17b38b003d'),\n 'plan_id': UUID('81df53ad-2159-4566-b43f-3c24d55d9d3e'),\n 'serialized_subscription': {'business_id': 'bbe18a98-d565-4423-a13a-1c17b38b003d',\n 'id': '977ac956-496f-4b54-8627-a0fd4fc3e9df',\n 'plan_id': '81df53ad-2159-4566-b43f-3c24d55d9d3e',\n 'status': 'active',\n 'user_id': 'ade39122-70b9-47f8-94e9-6ab4926e7398'},\n 'subscription': Subscription(id=UUID('977ac956-496f-4b54-8627-a0fd4fc3e9df'),\n plan_id=UUID('81df53ad-2159-4566-b43f-3c24d55d9d3e'),\n user_id=UUID('ade39122-70b9-47f8-94e9-6ab4926e7398'),\n business_id=UUID('bbe18a98-d565-4423-a13a-1c17b38b003d'),\n status=\u003cSubscriptionStatus.ACTIVE: 'active'\u003e),\n 'user_id': UUID('ade39122-70b9-47f8-94e9-6ab4926e7398')}\n```\n\nAs you see from the output, we've captured all the fields from the `create_subscription` method.\n\nLet's add a visual to concretize the concept.\n\n![](https://cdn.hashnode.com/res/hashnode/image/upload/v1718937071544/663c1e0b-b193-4233-b1d7-53e58492e5b1.png align=\"center\")\n\nWhen we call `SubscriptionAPI.create_subscription`, Python pushes a frame unto the stack. That frame contains the locals of our method.\n\nSounds good! We're making progress.\n\n## Step 2 - Capture the locals from functions we call\n\nWe don't only need to capture the locals from the `SubscriptionAPI` call, but all the other components (e.g. services, repositories, clients) it calls to satisfy our request. The `SubscriptionAPI` will call the `SubscriptionService` to create the subscription. The `SubscriptionService` will in turn, call the repository (`SubscriptionRepository`) to convince the database to create the data.\n\nLet's look at a quick sequence diagram to concretize the picture in our minds.\n\n![](https://cdn.hashnode.com/res/hashnode/image/upload/v1718937505858/36b51f30-f97b-4445-a337-6e65e921caf2.png align=\"center\")\n\nThankfully, Python can help us here too! Python captures all the variables for all the functions that get called. We learned earlier that each time Python invokes a function, it creates a stack frame with the locals. When a function calls another function, Python creates a new stack frame for the new function and its locals. So as functions are calling other functions, Python "])</script><script>self.__next_f.push([1,"pushes new frames unto the stack. As such, we just need to loop through each frame to capture all the variables.\n\nLet's look at the code for each of the components. They're mostly passthroughs.\n\n```python\nclass SubscriptionRepository:\n\n def create_subscription(\n self,\n ctx: Context,\n plan_id: UUID,\n user_id: UUID,\n business_id: UUID,\n status: SubscriptionStatus,\n ):\n # Pretend we're calling the ORM to create the subscription\n # Serialize the output as a Subcription value object\n return Subscription(...)\n\n\nclass SubscriptionService:\n\n def create_subscription(\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n ) -\u003e Subscription:\n status = SubscriptionStatus.ACTIVE\n return SubscriptionRepository().create_subscription(...)\n\nclass SubscriptionAPI:\n\n def create_subscription(\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n ):\n # Call the service to create a subscription\n subscription = SubscriptionService().create_subscription(...)\n # Serialize the subscription to JSON for return\n serialized_subscription = {...}\n\n return serialized_subscription\n```\n\nIf we were to look at the stack, when the `SubscriptionRepository.create_subscription` function is being interpreted, the stack would contain frames for the calling components (i.e. API and service).\n\n![](https://cdn.hashnode.com/res/hashnode/image/upload/v1718938824076/1699c24c-4493-4a4d-9d7f-98408a549038.png align=\"center\")\n\nFirst, there's a frame for the current method (i.e. `SubscriptionRepository.create_subscription`) with all its variables. It was called by `SubscriptionService.create_subscription`, so there's a stack for that. Finally, the service was called by `SubscriptionAPI.create_subscription` so there's a frame for that one as well.\n\nLet's look at how we'd achieve that.\n\n```python\ndef get_variables_from_all_calls() -\u003e dict:\n \"\"\"Get all the variables in the method that called us.\"\"\"\n local_variables = {}\n stack = inspect.stack()\n # We don't want the current frame, that's this function\n all_frames_except_this_one = stack[1:]\n for frame_information in all_frames_except_this_one:\n frame = frame_information.frame\n local_variables.update(frame.f_locals)\n return local_variables\n\n\nclass SubscriptionRepository:\n\n def create_subscription(\n self,\n ctx: Context,\n plan_id: UUID,\n user_id: UUID,\n business_id: UUID,\n status: SubscriptionStatus,\n ):\n # Call the ORM to create the subscription\n # Update the context with variables from all calls in the stack\n ctx.update(get_variables_from_all_calls()) \n # Serialize the output as a Subcription object\n return Subscription(...)\n```\n\nIn this example, I'm introducing a helper (`get_variables_from_all_calls`) to move the logic outside of our main function. Since the repository will call this method, that means Python will add a frame to the stack for this helper function as well. As such, we need to get all the frames except the current one. This explains why we're only capturing frames from index `1` and beyond. Finally, in `SubscriptionRepository.create_subscription` we're calling `get_variables_from_all_calls` to get all the variables for us.\n\nWhen we run the code, the `ctx` has the following contents:\n\n```python\n{'business_id': UUID('3498a3df-5001-481c-b0b8-a007609fb93c'),\n 'plan_id': UUID('fefbbafd-0f59-49bb-9255-3c09eef38fec'),\n 'status': \u003cSubscriptionStatus.ACTIVE: 'active'\u003e,\n 'user_id': UUID('de19ca36-67c5-48eb-80c8-70b0db9575ac')}\n```\n\nIf you're paying close attention, you'll notice this list is a little shorter than the last time we printed out the `Context`. In particular, this is missing the `serialized_subscription` and `subscription` object found in the `SubscriptionAPI.create_subscription` method. That happens because when we're capturing the variables, those don't exist yet.\n\nSee below if you're not convinced.\n\n```python\nclass Subs"])</script><script>self.__next_f.push([1,"criptionAPI:\n\n def create_subscription(\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n ):\n # The subscription variable is defined AFTER the call returns\n subscription = SubscriptionService().create_subscription(...)\n # Similarly, this is also defined AFTER the call returns\n serialized_subscription = {...}\n\n return serialized_subscription\n```\n\nWe'll keep going! That's a small problem in the grand scheme of things. As you'll see in the next step... we've got bigger problems.\n\n## Step 3 - Don't touch the code\n\nOne of our primary goals was to enable telemetry without manual effort. If we need to be adding a bunch of `get_variables_from_all_calls` calls all over our code, that simply won't work. It's less work than manually adding every variable, but still too much effort. Additionally, we don't want folks to modify their code to include these calls. We want these utilities to be mostly invisible but providing all the benefits necessary.\n\nLet's iterate and see what we can accomplish...\n\nWhat if we tried capturing the stack variables after the we made the function call.\n\n```python\nsubscription_api = SubscriptionAPI()\nctx = {}\n\nsubscription_api.create_subscription(\n ctx, plan_id, user_id, business_id)\n\n# Capture all variables\nget_variables_from_all_calls()\n```\n\nIf we print out the context of the `Context`, we'd get the following:\n\n```python\n{}\n```\n\nThe `Context` is empty. Yeah, that's right... it's completely empty.\n\nHere comes another important detail I omitted before...\n\n\u003e **Once a function is done executing, Python pops its frame from the stack.**\n\nSo... if you were to look at the stack at this point, it'd be empty (technically there's a \"module\" frame, but I've omitted that for brevity). There are no other frames on the stack, because after the function calls have exited, the frames associated with each call are completely gone. All of the previous approaches worked because we were capturing variables while we were *in* the call stack. Once those functions have exited we're no longer in the call stack and don't have access to those variables.\n\nSo we need a different approach. We need to capture variables while the relevant frames still exist on the stack. But how...\n\n## Step 4 - Byte off more than you can chew\n\nIt turns out, there's a way we can utilize the same approach, without modifying the code folks would write. We can introduce our capturing logic, at a lower-level... much lower. When we write Python code, it gets compiled into bytecode, which is then run by the interpreter.\n\n![](https://cdn.hashnode.com/res/hashnode/image/upload/v1719277675826/ba7ff392-856d-4314-b83b-2cfa8ad75c03.png align=\"center\")\n\nWe could *modify* the bytecode to accomplish the same objective. Python bytecode looks very different from Python. It's a sequence of instructions for the Python virtual machine. These instructions manipulate the internal stacks that Python uses. By modifying the resulting bytecode we can inject our logic and have the compiler execute our logic and capture the variables.\n\nTo exactly replicate what we did at a higher level, we need to capture the stack just before the function returns. So, we'd need to parse the bytecode instructions, looking for a return statement and then inject our logic right before that. We'd insert our logic to call the `get_variables_from_all_calls` function which would extract the variables from the stack. Then finally we'd relace the function's bytecode with our patched version. Sounds simple enough, right?\n\nThere are lots of different bytecode instructions. The following are the ones we're interested in:\n\n* `LOAD_GLOBAL` - this instruction enables us to make our function available for calling. This instruction tells Python to look up an object in the global namespace and place it on the evaluation stack. We'll use this to make our function ready for calling.\n \n* `LOAD_FAST` - this instruction enables us to provide arguments for a function call. This pushes a variable unto the stack for evaluation.\n \n* `CALL_FUNCTION`"])</script><script>self.__next_f.push([1," - this instruction is used to execute our function call. With this instruction we can also provide a number of positional arguments that should be popped and passed to the function.\n \n* `POP_TOP` - this instruction allows us to remove our function from the stack to continue the flow of the program. We'll use this to pop our function after we've called it.\n \n\nNext, I need to explain an idiosyncrasy we have to be familiar with to make this all work. Sometimes instructions spans multiple lines. We only need to add our patching instructions once. As such, we need to keep track of whether we've patched a line already so we don't do it multiple times.\n\nArmed with that knowledge let's examine the function that does the patching.\n\n```python\ndef patch_function_to_capture_stack_variables(func):\n # Get the bytecode that we plan to patch\n code = func.__code__\n bytecode = Bytecode.from_code(code)\n\n # Get a list of all line numbers for return instructions\n # Recall instructions can span multiple lines\n return_line_numbers = [\n instruction.lineno\n for instruction in bytecode\n if instruction.name == \"RETURN_VALUE\"\n ]\n\n # For each return instruction, we want to insert a call to capture_stack\n for return_line_number in return_line_numbers:\n # construct the instructions we need\n # Add our function to the stack\n add_capture_stack_function_to_the_stack = Instr(\"LOAD_GLOBAL\", \"get_variables_from_all_calls\")\n # Add the ctx as an argument for the function\n add_ctx_as_an_argument = Instr(\"LOAD_FAST\", \"ctx\")\n # Call our function and pop 1 arguemnt (ctx) to be passed to the function\n call_the_capture_stack_function = Instr(\"CALL_FUNCTION\", 1)\n # Remove our function to continue normal processing\n remove_the_capture_stack_function_from_the_stack = Instr(\"POP_TOP\")\n # Put the instructions all together\n catpure_stack_instructions = [\n add_capture_stack_function_to_the_stack,\n add_ctx_as_an_argument,\n call_the_capture_stack_function,\n remove_the_capture_stack_function_from_the_stack,\n ]\n\n for i, instruction in enumerate(bytecode):\n if instruction.lineno == return_line_number:\n bytecode[i:i] = catpure_stack_instructions\n # There could be other instructions on the same line\n # We don't want to insert the new instructions multiple times\n # So break out of the loop\n break\n\n updated_code = bytecode.to_code()\n\n func.__code__ = updated_code\n```\n\nAs shown in the code above, we're getting the bytecode, adding instructions to call our function and then applying the patch. We start by getting the `__code__` object of the function. That's the bytecode! We're using the `bytecode` library to simplify going to and from bytecode and construction of instructions. Next, we find all the lines that have return instructions. Finally, we construct our new instructions and add them to the bytecode.\n\nFinally, we patch the `SubscriptionRepository.create_subscription` function so it'll call our function.\n\n```python\npatch_function_to_capture_stack_variables(\n SubscriptionRepository.create_subscription)\n\nsubscription_api = SubscriptionAPI()\nsubscription_api.create_subscription(\n ctx, plan_id, user_id, business_id)\n\nprint(ctx)\n```\n\nRunning this code, gets us the following output:\n\n```python\n {'business_id': UUID('276dac6c-68ad-48bf-8a55-ac7ee6df64a5'),\n 'plan_id': UUID('e72b5c80-63d7-4c5a-a2a9-d4b5eef937e0'),\n 'status': \u003cSubscriptionStatus.ACTIVE: 'active'\u003e,\n 'user_id': UUID('946bf774-e95a-488d-b5b5-55a255aa5d2a')}\n```\n\nSuccess! Without asking anyone to modify their code, we were able to capture the local variables from the stack! This provides significant gains as we're able to reduce the manual effort involved. It's leaps and bounds further than where we started.\n\nYou'll notice though that it's the same output as step 3. It still has the shortcoming where it doesn't capture variables that have"])</script><script>self.__next_f.push([1,"n't been defined yet. There are ways to overcome this that don't take too much effort and don't re-introduce manual effort. Anyway... that's a story for another time.\n\n## Step 5 - Choke\n\nI've had to acknowledge that despite the energy rush this investigation brought, its complexity is well beyond our company's maintenance capacity. None of us are experts in the topic of bytecode. Theoretically, we're not doing much -- we're simply hooking into the code at a lower level. We wanted to avoid folks littering their code with our hooking logic, so we moved down an abstraction layer or two. That descent was unfortunately dangerously close to 6 feet under. Writing our Observability tooling hooks in bytecode would be suicidal for our tooling's life.\n\nWhat's worse, is that the implementation I've described thus far is woefully inadequate. We'd need to make several improvements and additions to *begin* the journey to production. We'd need some more bytecode instructions to import our library from elsewhere (in the script it's defined in the same file). Discover and account for additional idiosyncrasies of bytecode. We haven't addressed the multiple ways that functions can exit (e.g. early returns, unexpected exceptions and try-catch scenarios). Additionally, we would want to make our code more robust. We don't want failures in our library to cascade to the rest of the application. We'd probably need to catch all failures in our calls and properly log them (e.g. Sentry, SumoLogic, DataDog, etc...), or capture with metrics. Then, finally, we haven't thought about optimization or security. Who knows what challenges lie in that space!\n\nAnd so, we've started investigating alternatives. We've got some basic ideas we're toying around. One idea is to reduce the complexity, by only capturing function inputs and outputs. We could use a decorator-based approach to wrap public methods of all our components and capture the variables. We can implement our wrapping logic in a metaclass that we use to initialize our different components (e.g. services, repositories, clients, etc...). It's early days and we're still exploring things. I'll return to blog about our success (let's hope!) once we've found a working solution.\n\n## Conclusion\n\nIn our attempt to automate telemetry capture using Python, we explored the ambitious idea of introducing hooks at the bytecode level to automate this process. While our manual approach of enriching telemetry within our services has proven effective in reducing incident investigation time, scaling this across multiple teams and microservices was impractical. Our journey involved understanding Python's stack and bytecode intricacies to capture local variables across function calls. Despite the surge and adrenaline of this approach, the unavoidable complexity and maintenance challenges, lead us to abandon this method. We're now considering simpler, more maintainable alternatives like using decorators to capture function inputs and outputs.\n\nThanks for reading! If you've got thoughts on this topic, I'd love to hear it! Feel free to reach out to me on [Twitter](https://x.com/kramnaej) or [LinkedIn](https://www.linkedin.com/in/jean-mark-wright/)!\n\nAlso, many thanks to the [DataDog library](https://github.com/DataDog/datadogpy) I reverse-engineered to get an idea of how this could be accomplished.\n\nHere are a few resources that were helpful, if you're looking to learn more about bytecode.\n\n* [Demystifying Python Bytecode: A Guide to Understanding and Analyzing Code Execution](https://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1)\n \n* [DataDog Python Library](https://github.com/DataDog/datadogpy)\n \n\n[https://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1](https://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1)b:[\"$\",\"main\",null,{\"id\":\"main-content\",\"className\":\"min-h-screen bg-transparent\",\"child"])</script><script>self.__next_f.push([1,"ren\":[[\"$\",\"$L17\",null,{\"publication\":{\"__typename\":\"Publication\",\"id\":\"625387b175dccb6f343206bb\",\"url\":\"https://jaywhy13.hashnode.dev\",\"canonicalURL\":\"https://jaywhy13.hashnode.dev\",\"urlPattern\":\"simple\",\"title\":\"Perspective Unspoken\",\"displayTitle\":null,\"hasBadges\":true,\"descriptionSEO\":\"Welcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for \",\"seo\":{\"title\":null,\"description\":null,\"__typename\":\"SEO\"},\"publicMembers\":{\"totalDocuments\":1,\"__typename\":\"PublicMembers\"},\"about\":{\"html\":\"\u003cp\u003eWelcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.\u003c/p\u003e\\n\",\"text\":\"Welcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.\\n\",\"__typename\":\"Content\"},\"features\":{\"proTeam\":{\"isEnabled\":false,\"__typename\":\"ProTeamFeature\"},\"newsletter\":{\"isEnabled\":true,\"__typename\":\"NewsletterFeature\"},\"viewCount\":{\"isEnabled\":false,\"__typename\":\"ViewCountFeature\"},\"readTime\":{\"isEnabled\":true,\"__typename\":\"ReadTimeFeature\"},\"textSelectionSharer\":{\"isEnabled\":true,\"__typename\":\"TextSelectionSharerFeature\"},\"gptBotCrawling\":{\"__typename\":\"GPTBotCrawlingFeature\",\"isEnabled\":false},\"headlessCMS\":{\"isEnabled\":false,\"__typename\":\"HeadlessCMSFeature\"},\"__typename\":\"PublicationFeatures\"},\"aeoSettings\":{\"isEnabled\":false,\"llmsTxtEnabled\":true,\"crawlers\":{\"gptBot\":false,\"claudeBot\":true,\"perplexityBot\":true,\"googleExtended\":true,\"ccBot\":true,\"__typename\":\"AeoCrawlerSettings\"},\"__typename\":\"PublicationAeoSettings\"},\"metaTags\":null,\"ogMetaData\":{\"image\":null,\"__typename\":\"OpenGraphMetaData\"},\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"name\":\"Jean-Mark Wright\",\"username\":\"jaywhy13\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\"},\"preferences\":{\"__typename\":\"PublicationPreferences\",\"logo\":null,\"darkMode\":{\"__typename\":\"DarkModePreferences\",\"logo\":null,\"enabled\":false},\"navbarItems\":[{\"__typename\":\"PublicationNavbarItem\",\"id\":\"66e232819d27b6c7f345ab6e\",\"label\":\"Calendar\",\"url\":\"https://calendar.google.com/calendar/u/0/appointments/AcZssZ3tWQd0YSITufTZaAitKeE6Ln2NBOouN5Qlop4=?gv=true\",\"type\":\"link\",\"series\":null,\"page\":null}],\"enabledPages\":{\"__typename\":\"EnabledPages\",\"badges\":true,\"newsletter\":false,\"members\":false},\"layout\":\"magazine\",\"disableFooterBranding\":false,\"isSubscriptionModalDisabled\":false},\"favicon\":null,\"headerColor\":\"#ff8484\",\"integrations\":{\"fbPixelID\":null,\"fathomSiteID\":null,\"fathomCustomDomainEnabled\":false,\"fathomCustomDomain\":null,\"hotjarSiteID\":null,\"matomoSiteID\":null,\"matomoURL\":null,\"gaTrackingID\":null,\"gTagManagerID\":null,\"plausibleAnalyticsEnabled\":false,\"wmPaymentPointer\":null,\"koalaPublicKey\":null,\"msClarityID\":null,\"openPanelClientId\":null,\"consentManagement\":{\"enabled\":false,\"version\":1,\"privacyPolicyURL\":null,\"__typename\":\"ConsentManagement\"},\"__typename\":\"PublicationIntegrations\"},\"imprintV2\":null,\"postsCount\":{\"totalDocuments\":19,\"__typename\":\"PostConnection\"},\"isTeam\":true,\"links\":{\"twitter\":\"https://twitter.com/kramnaej\",\"instagram\":null,\"github\":\"https://github.com/jaywhy13\",\"website\":null,\"hashnode\":\"https://hashnode.com/@jaywhy13\",\"youtube\":null,\"dailydev\":null,\"linkedin\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"mastodon\":null,\"facebook\":null,\"bluesky\":null,\"__typename\":\"PublicationLinks\"},\"domainInfo\":{\"__typename\":\"DomainInfo\",\"hashnodeSubdomain\":\"jaywhy13\",\"domain\":null,\"wwwPrefixedDomain\":null},\"redirectionRules\":[{\"source\":\"/two-inportant-lessons-i-learnt-rolling-out-a-new-platform-feature\",\"destination\":\"/two-important-lessons-i-learned-rolling-out-a-new-platform-feature\",\"type\":301,\"__typename\":\"RedirectionRule\"},{\"source\":\"/would-sherlock-use-traces-or-metrics-to-debug-your-application\",\"destination\":\"/3-reasons-traces-better-than-metrics-for-debugging-your-application\",\"type\":30"])</script><script>self.__next_f.push([1,"1,\"__typename\":\"RedirectionRule\"}],\"customRules\":[{\"type\":302,\"path\":\"/linkedin\",\"destination\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"__typename\":\"CustomRule\"}],\"totalRecommendedPublications\":0,\"allowContributorEdits\":false,\"widgets\":[],\"post\":{\"__typename\":\"Post\",\"id\":\"667a1af4d2255d267d58e1df\",\"cuid\":\"clxtpx96y000109l8c9ry81qu\",\"title\":\"Automated telemetry capture via Python bytecode modification\",\"subtitle\":\"The pursuit of capturing application telemetry automatically\",\"slug\":\"automated-telemetry-capture-via-python-bytecode-modification\",\"brief\":\"Introduction\\nEver heard the expression \\\"Don't byte off more than you can chew\\\"? Today, I've got one of those stories about our attempt to capture telemetry automatically using Python by introducing hooks at the bytecode level. Our company has several...\",\"featured\":false,\"publishedAt\":\"2024-06-25T01:18:44.170Z\",\"updatedAt\":null,\"url\":\"https://jaywhy13.hashnode.dev/automated-telemetry-capture-via-python-bytecode-modification\",\"readTimeInMinutes\":15,\"views\":1586,\"reactionCount\":0,\"responseCount\":0,\"replyCount\":0,\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"name\":\"Jean-Mark Wright\",\"username\":\"jaywhy13\",\"deactivated\":false,\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"bio\":{\"html\":\"\u003cp\u003eJean-Mark is an articulate, customer-focused, visionary with extensive industry experience encompassing front and back end development in all phases of software development. He lives \\\"outside the box\\\" in pursuit of architectural elegance and relevant, tangible solutions. He is a strong believer that team empowerment fosters genius and is versed at working solo.\u003c/p\u003e\\n\",\"__typename\":\"Content\"},\"socialMediaLinks\":{\"website\":\"\",\"github\":\"https://github.com/jaywhy13\",\"twitter\":\"https://twitter.com/kramnaej\",\"facebook\":\"\",\"stackoverflow\":\"\",\"linkedin\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"__typename\":\"SocialMediaLinks\"}},\"coAuthors\":[],\"seo\":{\"title\":\"Automating Telemetry Capture in Python using Bytecode\",\"description\":\"Discover our journey of automating telemetry capture in Python using bytecode hooks, the challenges we faced, and the alternatives we're considering\",\"__typename\":\"SEO\"},\"coverImage\":{\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg\",\"isPortrait\":false,\"attribution\":\"https://unsplash.com/@markusspiske\",\"isAttributionHidden\":true,\"photographer\":\"Markus Spiske\",\"__typename\":\"CoverImage\"},\"content\":{\"html\":\"$18\",\"markdown\":\"$19\",\"__typename\":\"Content\"},\"preferences\":{\"pinnedToBlog\":false,\"disableComments\":false,\"stickCoverToBottom\":false,\"isDelisted\":false,\"__typename\":\"PostPreferences\"},\"series\":null,\"tags\":[{\"id\":\"56744721958ef13879b94d67\",\"slug\":\"python\",\"name\":\"Python\",\"__typename\":\"Tag\"},{\"id\":\"5c513528a8f986a7229128d9\",\"slug\":\"bytecode\",\"name\":\"bytecode\",\"__typename\":\"Tag\"},{\"id\":\"5f1df3b4f4016901885ccc96\",\"slug\":\"observability\",\"name\":\"observability\",\"__typename\":\"Tag\"},{\"id\":\"632ae84c0b80d2f3c43f152b\",\"slug\":\"telemetry\",\"name\":\"telemetry\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b948a2\",\"slug\":\"microservices\",\"name\":\"Microservices\",\"__typename\":\"Tag\"},{\"id\":\"5fc2287b55b779631e1acc29\",\"slug\":\"instrumentation\",\"name\":\"instrumentation\",\"__typename\":\"Tag\"}],\"ogMetaData\":{\"image\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg\",\"__typename\":\"OpenGraphMetaData\"},\"canonicalUrl\":null,\"hasLatexInPost\":false,\"faq\":[],\"features\":{\"tableOfContents\":{\"isEnabled\":false,\"items\":[],\"__typename\":\"TableOfContents\"},\"__typename\":\"PostFeatures\"}},\"posts\":{\"edges\":[{\"node\":{\"__typename\":\"Post\",\"id\":\"694a1633103f879a13892085\",\"title\":\"Instrumenting at 10 years per second\",\"slug\":\"instrumenting-at-10-years-per-second\",\"publishedAt\":\"2025-12-23T04:10:27.099Z\",\"cuid\":\"cmji2h6zf000102jx6k7x1e0n\",\"url\":\"https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second\",\"subtitle\":\"How we made custom instrumentation blazing fast, simple, and data-centric\",\"brief\":\"Intr"])</script><script>self.__next_f.push([1,"oduction\\nI've found a new love - custom instrumentation, and I can’t keep it to myself. We manually increased the custom instrumentation in one of our newer services and loved the impact. The instrumentation accelerated forensic investigations, r...\",\"readTimeInMinutes\":8,\"views\":276,\"tags\":[{\"id\":\"5f1df3b4f4016901885ccc96\",\"slug\":\"observability\",\"name\":\"observability\",\"__typename\":\"Tag\"},{\"id\":\"5fc2287b55b779631e1acc29\",\"slug\":\"instrumentation\",\"name\":\"instrumentation\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b94d67\",\"slug\":\"python\",\"name\":\"Python\",\"__typename\":\"Tag\"},{\"id\":\"56744723958ef13879b95342\",\"slug\":\"python3\",\"name\":\"Python 3\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"67b2ab28c4ef92b38e5bdda0\",\"title\":\"On shipping smaller PRs\",\"slug\":\"on-shipping-smaller-prs\",\"publishedAt\":\"2025-02-17T03:21:12.558Z\",\"cuid\":\"cm78hnn8u000409l51hurfg9f\",\"url\":\"https://jaywhy13.hashnode.dev/on-shipping-smaller-prs\",\"subtitle\":\"Concrete examples for how to ship smaller PRs\",\"brief\":\"Introduction\\nIn my last I talked about the importance of shipping smaller PRs. In this article I’ll give some practical examples for how PRs can be split into smaller chunks. The examples aren’t organized into specific themes. Rather they’re arbitrar...\",\"readTimeInMinutes\":8,\"views\":83,\"tags\":[{\"id\":\"59c5f7f0ea46c0424704e8c1\",\"slug\":\"pull-requests\",\"name\":\"Pull Requests\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b949f9\",\"slug\":\"code-review\",\"name\":\"code review\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b94ad1\",\"slug\":\"software-development\",\"name\":\"software development\",\"__typename\":\"Tag\"},{\"id\":\"569d22c892921b8f79d35f68\",\"slug\":\"software-engineering\",\"name\":\"Software Engineering\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/o9oQaOGpLz0/upload/9e19fbfd378d65c448f7233b2a273b79.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"676b90a8cb76c2499dc67afb\",\"title\":\"Small PRs ship faster - QED\",\"slug\":\"small-prs-ship-faster-qed\",\"publishedAt\":\"2024-12-25T04:57:12.349Z\",\"cuid\":\"cm53fb3j1000309mb1rckcb01\",\"url\":\"https://jaywhy13.hashnode.dev/small-prs-ship-faster-qed\",\"subtitle\":\"My unrestrained passion for shipping smaller PRs constrained into a handful of benefits\",\"brief\":\"Introduction\\nWelcome to my very scientific paper where I propose that small PRs ship faster than big ones. I will construct my hypothesis with carefully crafted examples, concrete evidence and lots of words and sentences in general. I guarantee there...\",\"readTimeInMinutes\":5,\"views\":110,\"tags\":[{\"id\":\"56744721958ef13879b949f9\",\"slug\":\"code-review\",\"name\":\"code review\",\"__typename\":\"Tag\"},{\"id\":\"59c5f7f0ea46c0424704e8c1\",\"slug\":\"pull-requests\",\"name\":\"Pull Requests\",\"__typename\":\"Tag\"},{\"id\":\"5d9352eae235795f6eb6df65\",\"slug\":\"pull-request\",\"name\":\"pull request\",\"__typename\":\"Tag\"},{\"id\":\"56744723958ef13879b953b4\",\"slug\":\"review\",\"name\":\"review\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/JzMS3RLSZn8/upload/a18ee8f82d0d5"])</script><script>self.__next_f.push([1,"6644ab82f6c1bead8b4.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"671072daa4aac495b3ea5d5a\",\"title\":\"Order matters - making a compound index 50x faster\",\"slug\":\"order-matters-making-a-compound-index-50x-faster\",\"publishedAt\":\"2024-10-17T02:13:46.740Z\",\"cuid\":\"cm2co35h0000009ld0dg42agd\",\"url\":\"https://jaywhy13.hashnode.dev/order-matters-making-a-compound-index-50x-faster\",\"subtitle\":\"A story of how we made a compound index 50x faster by re-organizing the fields\",\"brief\":\"Introduction\\nToday I’ll talk about an endpoint that initially performed under 30ms, then crept up to ~500ms after a couple months. Our investigation revealed that a query, powered by a compound index was responsible for the elevated latency. We explo...\",\"readTimeInMinutes\":6,\"views\":1082,\"tags\":[{\"id\":\"56744722958ef13879b950eb\",\"slug\":\"databases\",\"name\":\"Databases\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b949b5\",\"slug\":\"postgresql\",\"name\":\"PostgreSQL\",\"__typename\":\"Tag\"},{\"id\":\"56744722958ef13879b94f0b\",\"slug\":\"postgres\",\"name\":\"postgres\",\"__typename\":\"Tag\"},{\"id\":\"64308297311bf43ae8efe880\",\"slug\":\"postgresql-performance\",\"name\":\"Postgresql-performance \",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b94821\",\"slug\":\"optimization\",\"name\":\"optimization\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1726107899370/04b1ac6d-7825-4a05-b5d0-0be78a2f49d9.png\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"66e239b9dcb487d824fc1b6a\",\"title\":\"Dear Editor: We need better Database Observability\",\"slug\":\"dear-editor-we-need-better-database-observability\",\"publishedAt\":\"2024-09-12T00:45:45.852Z\",\"cuid\":\"cm0ykj5do000708lj0q3laq8c\",\"url\":\"https://jaywhy13.hashnode.dev/dear-editor-we-need-better-database-observability\",\"subtitle\":\"A call for better database observability, or awareness to a better reality\",\"brief\":\"Introduction\\nDear Editor, hear my plea. I pen this piece in earnest search of enlightenment, or the beginnings of a well needed change; we need better database observability! Specifically, more granular, and useful database spans. Coarse, high-level ...\",\"readTimeInMinutes\":8,\"views\":745,\"tags\":[{\"id\":\"5f1df3b4f4016901885ccc96\",\"slug\":\"observability\",\"name\":\"observability\",\"__typename\":\"Tag\"},{\"id\":\"56744722958ef13879b950eb\",\"slug\":\"databases\",\"name\":\"Databases\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/NvFkYV2ngOk/upload/2f4cbc39f32355a334d9119af2dd086c.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"}],\"__typename\":\"PostConnection\"},\"redirectedPost\":null,\"staticPage\":null},\"posts\":[\"$b:props:children:0:props:publication:post\"],\"host\":\"jaywhy13.hashnode.dev\"}],\"$L1a\",\"$L1b\",null,\"$L1c\",\"$L1d\",null,\"$L1e\"]}]\n1f:I[62031,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"ReadingModeToggle\"]\n20:I[81356,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"75"])</script><script>self.__next_f.push([1,"63\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"Image\"]\n21:I[52619,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"8974\",\"static/chunks/app/page-cc18ae1a3a7a5ed8.js\"],\"\"]\n22:I[33884,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"Avatar\"]\n23:I[33884,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"AvatarImage\"]\n24:I[33884,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"AvatarFallback\"]\n27:I[51285,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"CommentsSection\"]\n28:I[20730,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"default\"]\n1a:[\"$\",\"section\",null,{\"className\":\"py-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"px-4 sm:px-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]\",\"children\":[\"$\",\"div\",null,{\"className\":\"grid gap-5 max-w-full\",\"children\":[\"$\",\"article\",null,{\"className\":\"w-full max-w-full "])</script><script>self.__next_f.push([1,"rounded-xl border bg-card text-card-foreground\",\"children\":[[\"$\",\"div\",null,{\"className\":\"px-5 pt-12 pb-8 sm:px-8\",\"children\":[\"$\",\"header\",null,{\"className\":\"space-y-5\",\"children\":[\"$\",\"div\",null,{\"className\":\"space-y-4 text-center max-w-4xl mx-auto\",\"children\":[[\"$\",\"h1\",null,{\"className\":\"text-2xl sm:text-3xl md:text-4xl lg:text-5xl font-bold leading-tight tracking-tight text-balance text-foreground\",\"children\":\"Automated telemetry capture via Python bytecode modification\"}],[\"$\",\"p\",null,{\"className\":\"text-lg sm:text-xl leading-normal font-normal text-muted-foreground text-balance\",\"children\":\"The pursuit of capturing application telemetry automatically\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center justify-center gap-3 text-sm text-muted-foreground\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-2\",\"children\":[[\"$\",\"span\",null,{\"children\":\"Published\"}],[\"$\",\"time\",null,{\"dateTime\":\"2024-06-25T01:18:44.170Z\",\"children\":\"June 25, 2024\"}]]}],[[\"$\",\"span\",null,{\"className\":\"text-muted-foreground\",\"children\":\"•\"}],[\"$\",\"span\",null,{\"children\":[15,\" min read\"]}]],[\"$\",\"span\",null,{\"className\":\"text-muted-foreground\",\"children\":\"•\"}],[\"$\",\"$L1f\",null,{\"slug\":\"automated-telemetry-capture-via-python-bytecode-modification\"}]]}]]}]}]}],[\"$\",\"div\",null,{\"className\":\"w-full\",\"children\":[\"$\",\"$L20\",null,{\"src\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg\",\"alt\":\"Automated telemetry capture via Python bytecode modification\",\"width\":1200,\"height\":750,\"className\":\"w-full h-auto\",\"priority\":true}]}],[\"$\",\"div\",null,{\"className\":\"px-5 py-12 sm:px-8 space-y-8\",\"children\":[[\"$\",\"div\",null,{\"className\":\"max-w-2xl mx-auto\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center gap-5\",\"children\":[[\"$\",\"div\",\"6253878075dccb6f343206b7\",{\"className\":\"flex items-center gap-5\",\"children\":[false,[\"$\",\"div\",null,{\"className\":\"flex gap-3 items-start\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/@jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener\",\"children\":[\"$\",\"$L22\",null,{\"className\":\"h-10 w-10 border border-border transition-opacity hover:opacity-80\",\"children\":[[\"$\",\"$L23\",null,{\"src\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg?auto=compress,format\u0026format=webp\",\"alt\":\"Jean-Mark Wright\"}],[\"$\",\"$L24\",null,{\"className\":\"bg-muted text-muted-foreground font-semibold text-lg\",\"children\":\"J\"}]]}]}],[\"$\",\"div\",null,{\"className\":\"min-w-0\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-2\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/@jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener\",\"className\":\"font-medium text-foreground hover:text-primary transition-colors\",\"children\":\"Jean-Mark Wright\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center gap-1.5 text-sm\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"https://twitter.com/kramnaej\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground hover:text-foreground transition-colors\",\"aria-label\":\"X (Twitter)\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-x-twitter\",\"aria-hidden\":true}]}],[\"$\",\"$L21\",null,{\"href\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground hover:text-foreground transition-colors\",\"aria-label\":\"LinkedIn\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-linkedin\",\"aria-hidden\":true}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"text-sm text-muted-foreground line-clamp-2\",\"dangerouslySetInnerHTML\":{\"__html\":\"\u003cp\u003eJean-Mark is an articulate, customer-focused, visionary with extensive industry experience encompassing front and back end development in all phases of software development. He lives \\\"outside the box\\\" in pursuit of architectural elegance and relevant, tangible solutions. He is a strong believer that team empowerment fosters genius and is versed at working solo.\u003c/p\u003e\\n\"}}]]}]]}]]}]]}]}],null,\"$L25\",false,\"$L26\"]}]]}]}]}]}]}]\n1b:[\"$\",\"section\",null,{\"className\":\"pb-8\",\"child"])</script><script>self.__next_f.push([1,"ren\":[\"$\",\"div\",null,{\"className\":\"px-4 sm:px-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]\",\"children\":[\"$\",\"div\",null,{\"className\":\"max-w-4xl w-full mx-auto\",\"children\":[\"$\",\"div\",null,{\"className\":\"rounded-xl border bg-card text-card-foreground p-5 sm:p-8\",\"children\":[\"$\",\"$L27\",null,{\"host\":\"jaywhy13.hashnode.dev\",\"slug\":\"automated-telemetry-capture-via-python-bytecode-modification\",\"postUrl\":\"https://hashnode.com/posts/automated-telemetry-capture-via-python-bytecode-modification/667a1af4d2255d267d58e1df\",\"initialEdges\":[],\"initialPageInfo\":{\"hasNextPage\":false,\"endCursor\":\"10\",\"totalDocuments\":0,\"__typename\":\"PageInfo\"},\"totalComments\":0}]}]}]}]}]}]\n1c:[\"$\",\"section\",null,{\"className\":\"pb-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"px-4 sm:px-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]\",\"children\":[\"$\",\"div\",null,{\"className\":\"max-w-4xl w-full mx-auto\",\"children\":[\"$\",\"section\",null,{\"className\":\"space-y-6\",\"children\":[[\"$\",\"h2\",null,{\"className\":\"text-2xl font-bold tracking-tight text-balance\",\"children\":\"More from this blog\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-col gap-6\",\"children\":[[\"$\",\"$L28\",\"694a1633103f879a13892085\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:0:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"67b2ab28c4ef92b38e5bdda0\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:1:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"676b90a8cb76c2499dc67afb\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:2:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"671072daa4aac495b3ea5d5a\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:3:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"66e239b9dcb487d824fc1b6a\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:4:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"div\",\"author-card\",{\"ref\":\"$undefined\",\"className\":\"rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 p-5 flex flex-col\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex flex-col items-center text-center gap-5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-col items-center gap-3\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex h-16 w-16 shrink-0 items-center justify-center rounded-full bg-muted\",\"children\":[\"$\",\"i\",null,{\"className\":\"text-2xl fa-brands fa-hashnode text-foreground\",\"aria-hidden\":true}]}],[\"$\",\"div\",null,{\"className\":\"space-y-1\",\"children\":[[\"$\",\"p\",null,{\"className\":\"text-base font-semibold text-foreground leading-tight\",\"children\":\"Perspective Unspoken\"}],[\"$\",\"p\",null,{\"className\":\"text-sm text-muted-foreground\",\"children\":[19,\" posts\"]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"prose prose-sm dark:prose-invert max-w-xs mx-auto text-muted-foreground\",\"dangerouslySetInnerHTML\":{\"__html\":\"\u003cp\u003eWelcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.\u003c/p\u003e\\n\"}}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center justify-center gap-3\",\"children\":[[],[[\"$\",\"a\",\"Twitter\",{\"href\":\"https://twitter.com/kramnaej\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"Twitter\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-x-twitter\",\"aria-hidden\":true}]}],[\"$\",\"a\",\"GitHub\",{\"href\":\"https://github.com/jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"GitHub\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-github\",\"aria-hidden\":true}]}],[\"$\",\"a\",\"LinkedIn\",{\"href\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"LinkedIn\",\"children\":[\"$\",\"i\""])</script><script>self.__next_f.push([1,",null,{\"className\":\"fa-brands fa-linkedin\",\"aria-hidden\":true}]}],[\"$\",\"a\",\"Hashnode\",{\"href\":\"https://hashnode.com/@jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"Hashnode\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-hashnode\",\"aria-hidden\":true}]}]]]}]]}]}]]}]]}]}]}]}]}]\n29:T4ed,{\"@context\":\"https://schema.org\",\"@type\":\"Article\",\"url\":\"https://jaywhy13.hashnode.dev/automated-telemetry-capture-via-python-bytecode-modification\",\"mainEntityOfPage\":\"https://jaywhy13.hashnode.dev/automated-telemetry-capture-via-python-bytecode-modification\",\"headline\":\"Automated telemetry capture via Python bytecode modification\",\"description\":\"Introduction\\nEver heard the expression \\\"Don't byte off more than you can chew\\\"? Today, I've got one of those stories about our attempt to capture telemetry automatically using Python by introducing hooks at the bytecode level. Our company has several...\",\"author\":{\"@type\":\"Person\",\"name\":\"Jean-Mark Wright\",\"url\":\"https://hashnode.com/@jaywhy13\",\"image\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"sameAs\":\"https://x.com/kramnaej\"},\"publisher\":{\"@type\":\"Organization\",\"name\":\"Perspective Unspoken\",\"url\":\"https://jaywhy13.hashnode.dev\"},\"datePublished\":\"2024-06-25T01:18:44.170Z\",\"dateModified\":\"2024-06-25T01:18:44.170Z\",\"image\":{\"@type\":\"ImageObject\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/FXFz-sW0uwo/upload/8f9e066075360b1875b88456be15e668.jpeg\"},\"keywords\":\"Python, bytecode, observability, telemetry, Microservices, instrumentation\"}1d:[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"$29\"}}]\n2a:I[78655,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"TableOfContents\"]\n2b:T871c,\u003ch2 id=\"heading-introduction\"\u003eIntroduction\u003c/h2\u003e\n\u003cp\u003eEver heard the expression \"Don't \u003cstrong\u003ebyte\u003c/strong\u003e off more than you can chew\"? Today, I've got one of those stories about our attempt to capture telemetry automatically using Python by introducing hooks at the \u003cem\u003ebytecode\u003c/em\u003e level. Our company has several teams managing 20+ microservices and thousands of endpoint. We have a successful manual telemetry capture pattern that significantly reduces incident investigation time by enriching and attaching telemetry to our tools like DataDog, Sentry, and Sumo. However, it's too time-consuming for widespread adoption across all teams. Our team implemented this pattern efficiently, but replicating it manually for other teams would take years! This lead us to explore and eventually \u003cem\u003eabandon\u003c/em\u003e an overly complex automated bytecode capture approach.\u003c/p\u003e\n\u003cp\u003eBelow is the story of that journey.\u003c/p\u003e\n\u003ch2 id=\"heading-motivation\"\u003eMotivation\u003c/h2\u003e\n\u003cp\u003eUltimately we want to increase our business uptime by reducing incident recovery time. We achieve this by enhancing our production experience through access to high quality telemetry for investigation. We're aiming to replicate a local debugger in production. Enhancing system state visibility accelerates our investigations, which reduces disruption time. If we're able to clearly see system state when looking at a trace, or an error, or log, we'll drastically reduce remediation time. High quality telemetry provides a solid base for hypotheses creation and testing.\u003c/p\u003e\n\u003cp\u003eWe've had successes that confirm that this approach works too. We had an incident where we inadvertently gave \u003ca target=\"_blank\" href=\"https://jaywhy13.hashnode.dev/solving-like-sherlock-a-15-minute-case-with-observabi"])</script><script>self.__next_f.push([1,"lity\" rel=\"noopener noreferrer nofollow ugc\"\u003efree access\u003c/a\u003e to some premium features. We got to our root cause for that issue in less than 15 minutes thanks to the tooling improvements. In that incident, we zoomed out from an individual trace and aggregated over a suspicious property which confirmed our theory. We had another incident where we fixed a broken Kafka worker and it caused data corruption for a running migration. Again, examining traces and inspecting system state was critical for understanding issues quickly.\u003c/p\u003e\n\u003ch2 id=\"heading-step-0-the-existing-manual-process\"\u003eStep 0 - The existing, manual process\u003c/h2\u003e\n\u003cp\u003eLet's take a look at what our manual process looks like. Below I'm showing the code for a \u003ccode\u003eSubscriptionAPI\u003c/code\u003e. This represents the highest component in our architecture. An API typically calls a service, then a repository for data operations. You'll see there's a \u003ccode\u003eContext\u003c/code\u003e (alias for a Python dictionary) that we start adding information to. That's us collecting telemetry. Below we're adding the input values to the \u003ccode\u003eContext\u003c/code\u003e like the \u003ccode\u003eapi\u003c/code\u003e, \u003ccode\u003euser_id\u003c/code\u003e, \u003ccode\u003ebusiness_id\u003c/code\u003e, \u003ccode\u003eplan_id\u003c/code\u003e (the subscription's plan) and eventually the \u003ccode\u003eserialized_subscription\u003c/code\u003e we generate as the API output. The \u003ccode\u003eContext\u003c/code\u003e is then passed down the hierarchy to all other calls. In this scenario, the \u003ccode\u003eSubscriptionAPI\u003c/code\u003e calls the \u003ccode\u003eSubscriptionService.create_subscription\u003c/code\u003e method. It passes the \u003ccode\u003ectx\u003c/code\u003e to that method so we can continue to populate it.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionAPI\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Start capturing telemetry (using a dictionary \"ctx\")\u003c/span\u003e\n ctx[\u003cspan class=\"hljs-string\"\u003e\"api\"\u003c/span\u003e] = \u003cspan class=\"hljs-string\"\u003e\"create_subscription\"\u003c/span\u003e\n ctx[\u003cspan class=\"hljs-string\"\u003e\"user_id\"\u003c/span\u003e] = str(user_id)\n ctx[\u003cspan class=\"hljs-string\"\u003e\"plan_id\"\u003c/span\u003e] = str(plan_id)\n ctx[\u003cspan class=\"hljs-string\"\u003e\"business_id\"\u003c/span\u003e] = str(business_id)\n\n \u003cspan class=\"hljs-comment\"\u003e# Call the service to create a subscription\u003c/span\u003e\n subscription = SubscriptionService().create_subscription(\n ctx, plan_id=plan_id, user_id=user_id, business_id=business_id\n )\n \u003cspan class=\"hljs-comment\"\u003e# Serialize the subscription to JSON for return\u003c/span\u003e\n serialized_subscription = {\n \u003cspan class=\"hljs-string\"\u003e\"id\"\u003c/span\u003e: str(subscription.id),\n \u003cspan class=\"hljs-string\"\u003e\"plan_id\"\u003c/span\u003e: str(subscription.plan_id),\n \u003cspan class=\"hljs-string\"\u003e\"user_id\"\u003c/span\u003e: str(subscription.user_id),\n \u003cspan class=\"hljs-string\"\u003e\"business_id\"\u003c/span\u003e: str(subscription.business_id),\n \u003cspan class=\"hljs-string\"\u003e\"status\"\u003c/span\u003e: subscription.status.value,\n }\n ctx[\u003cspan class=\"hljs-string\"\u003e\"serialized_subscription\"\u003c/span\u003e] = serialized_subscription\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e serialized_subscription\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eAs we populate the \u003ccode\u003ectx\u003c/code\u003e in each layer of the application, we end up with a rich description of our current system state.\u003c/p\u003e\n\u003cp\u003eBelow is an example of us invoke the \u003ccode\u003eSubscriptionAPI\u003c/code\u003e and then examining the \u003ccode\u003eContext\u003c/code\u003e.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-comment\"\u003e# Declare variables (plan_id, user_id, etc...)\u003c/span\u003e\nctx = {}\n\nsubscription_api = SubscriptionAPI()\nsubscription_api.create_subscription(\n ctx, plan_id, user_id, business_id)\n\nprint(ctx)\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThe output of the \u003ccode\u003eContext\u003c/code\u003e looks like this:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-json\"\u003e{\u003cspan class=\"hljs-attr\"\u003e\"api\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"create_subscription\"\u003c/span\u003e,\n \u003cspan"])</script><script>self.__next_f.push([1," class=\"hljs-attr\"\u003e\"business_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"bc7464cc-dd17-481e-88a4-93ba5711f2a1\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"plan_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"43a3ec11-ba1d-4fff-a958-f3945fc46a36\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"serialized_subscription\"\u003c/span\u003e: {\u003cspan class=\"hljs-attr\"\u003e\"business_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"bc7464cc-dd17-481e-88a4-93ba5711f2a1\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"f2dc517e-5c9c-4f55-8a26-ef43dd7bc0ef\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"plan_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"43a3ec11-ba1d-4fff-a958-f3945fc46a36\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"status\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"active\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"user_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"61346915-3c35-4ba4-87fd-7d092053c940\"\u003c/span\u003e},\n \u003cspan class=\"hljs-attr\"\u003e\"status\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"active\"\u003c/span\u003e,\n \u003cspan class=\"hljs-attr\"\u003e\"user_id\"\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e\"61346915-3c35-4ba4-87fd-7d092053c940\"\u003c/span\u003e}\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThis manual capture approach works well when you're starting fresh and you establish this as a team practice. However, when you've got thousands of endpoints scattered across multiple services without the pattern, we need an automated approach.\u003c/p\u003e\n\u003cp\u003eLet's dive into the steps we took on the journey to automated telemetry capture.\u003c/p\u003e\n\u003ch2 id=\"heading-step-1-capture-variables-in-a-method\"\u003eStep 1 - Capture variables in a method\u003c/h2\u003e\n\u003cp\u003eThe first step is understanding how we capture all the variables in a method. Once we can capture variables in one method, we can extend this approach to the rest of the architecture stack. Python provides a facility for accomplishing this. We can use \u003ccode\u003einspect.stack()\u003c/code\u003e to get local variables from a function. The stack stores variables for each function being called. Python creates a frame for each function being called. Let's see how this works in practice.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionAPI\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Call the service to create a subscription\u003c/span\u003e\n subscription = SubscriptionService().create_subscription(\n ctx, plan_id=plan_id, user_id=user_id, business_id=business_id\n )\n \u003cspan class=\"hljs-comment\"\u003e# Serialize the subscription to JSON for return\u003c/span\u003e\n serialized_subscription = {...}\n\n \u003cspan class=\"hljs-comment\"\u003e# Capture all the variables in the method\u003c/span\u003e\n stack = inspect.stack()\n local_variables = stack[\u003cspan class=\"hljs-number\"\u003e0\u003c/span\u003e].frame.f_locals\n\n \u003cspan class=\"hljs-comment\"\u003e# Capture all the local variables in the context\u003c/span\u003e\n ctx.update(local_variables)\n\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e serialized_subscription\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eNear the end of our \u003ccode\u003ecreate_subscription\u003c/code\u003e function, we're accessing the \u003ccode\u003estack\u003c/code\u003e and printing the locals in the first frame (i.e. at index \u003ccode\u003e0\u003c/code\u003e). This gives us all the local variables in the current function (i.e. \u003ccode\u003ecreate_subscription\u003c/code\u003e). Every time Python calls a function, it creates a frame for it and stores the locals.\u003c/p\u003e\n\u003cp\u003eIf we run the \u003ccode\u003ecreate_subscription\u003c/code\u003e method and examine the \u003ccode\u003ectx\u003c/code\u003e, it looks like this:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e{\u003cspan class=\"hljs-string\"\u003e'business_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'bbe18a98-d565-4423-a13a-1c17b38b003d'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'plan_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'81df53ad-2159-4566-b43f-3c24d55d9d3e'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'serialized_subscription'\u003c/span\u003e: "])</script><script>self.__next_f.push([1,"{\u003cspan class=\"hljs-string\"\u003e'business_id'\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e'bbe18a98-d565-4423-a13a-1c17b38b003d'\u003c/span\u003e,\n \u003cspan class=\"hljs-string\"\u003e'id'\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e'977ac956-496f-4b54-8627-a0fd4fc3e9df'\u003c/span\u003e,\n \u003cspan class=\"hljs-string\"\u003e'plan_id'\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e'81df53ad-2159-4566-b43f-3c24d55d9d3e'\u003c/span\u003e,\n \u003cspan class=\"hljs-string\"\u003e'status'\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e'active'\u003c/span\u003e,\n \u003cspan class=\"hljs-string\"\u003e'user_id'\u003c/span\u003e: \u003cspan class=\"hljs-string\"\u003e'ade39122-70b9-47f8-94e9-6ab4926e7398'\u003c/span\u003e},\n \u003cspan class=\"hljs-string\"\u003e'subscription'\u003c/span\u003e: Subscription(id=UUID(\u003cspan class=\"hljs-string\"\u003e'977ac956-496f-4b54-8627-a0fd4fc3e9df'\u003c/span\u003e),\n plan_id=UUID(\u003cspan class=\"hljs-string\"\u003e'81df53ad-2159-4566-b43f-3c24d55d9d3e'\u003c/span\u003e),\n user_id=UUID(\u003cspan class=\"hljs-string\"\u003e'ade39122-70b9-47f8-94e9-6ab4926e7398'\u003c/span\u003e),\n business_id=UUID(\u003cspan class=\"hljs-string\"\u003e'bbe18a98-d565-4423-a13a-1c17b38b003d'\u003c/span\u003e),\n status=\u0026lt;SubscriptionStatus.ACTIVE: \u003cspan class=\"hljs-string\"\u003e'active'\u003c/span\u003e\u0026gt;),\n \u003cspan class=\"hljs-string\"\u003e'user_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'ade39122-70b9-47f8-94e9-6ab4926e7398'\u003c/span\u003e)}\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eAs you see from the output, we've captured all the fields from the \u003ccode\u003ecreate_subscription\u003c/code\u003e method.\u003c/p\u003e\n\u003cp\u003eLet's add a visual to concretize the concept.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1718937071544/663c1e0b-b193-4233-b1d7-53e58492e5b1.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eWhen we call \u003ccode\u003eSubscriptionAPI.create_subscription\u003c/code\u003e, Python pushes a frame unto the stack. That frame contains the locals of our method.\u003c/p\u003e\n\u003cp\u003eSounds good! We're making progress.\u003c/p\u003e\n\u003ch2 id=\"heading-step-2-capture-the-locals-from-functions-we-call\"\u003eStep 2 - Capture the locals from functions we call\u003c/h2\u003e\n\u003cp\u003eWe don't only need to capture the locals from the \u003ccode\u003eSubscriptionAPI\u003c/code\u003e call, but all the other components (e.g. services, repositories, clients) it calls to satisfy our request. The \u003ccode\u003eSubscriptionAPI\u003c/code\u003e will call the \u003ccode\u003eSubscriptionService\u003c/code\u003e to create the subscription. The \u003ccode\u003eSubscriptionService\u003c/code\u003e will in turn, call the repository (\u003ccode\u003eSubscriptionRepository\u003c/code\u003e) to convince the database to create the data.\u003c/p\u003e\n\u003cp\u003eLet's look at a quick sequence diagram to concretize the picture in our minds.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1718937505858/36b51f30-f97b-4445-a337-6e65e921caf2.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eThankfully, Python can help us here too! Python captures all the variables for all the functions that get called. We learned earlier that each time Python invokes a function, it creates a stack frame with the locals. When a function calls another function, Python creates a new stack frame for the new function and its locals. So as functions are calling other functions, Python pushes new frames unto the stack. As such, we just need to loop through each frame to capture all the variables.\u003c/p\u003e\n\u003cp\u003eLet's look at the code for each of the components. They're mostly passthroughs.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionRepository\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self,\n ctx: Context,\n plan_id: UUID,\n user_id: UUID,\n business_id: UUID,\n status: SubscriptionStatus,\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Pretend we're calling the ORM to create the subscription\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Serialize the output as a Subcription value object\u003c/span\u003e\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e Subscription("])</script><script>self.__next_f.push([1,"...)\n\n\n\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionService\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n \u003c/span\u003e) -\u0026gt; Subscription:\u003c/span\u003e\n status = SubscriptionStatus.ACTIVE\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e SubscriptionRepository().create_subscription(...)\n\n\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionAPI\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Call the service to create a subscription\u003c/span\u003e\n subscription = SubscriptionService().create_subscription(...)\n \u003cspan class=\"hljs-comment\"\u003e# Serialize the subscription to JSON for return\u003c/span\u003e\n serialized_subscription = {...}\n\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e serialized_subscription\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eIf we were to look at the stack, when the \u003ccode\u003eSubscriptionRepository.create_subscription\u003c/code\u003e function is being interpreted, the stack would contain frames for the calling components (i.e. API and service).\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1718938824076/1699c24c-4493-4a4d-9d7f-98408a549038.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eFirst, there's a frame for the current method (i.e. \u003ccode\u003eSubscriptionRepository.create_subscription\u003c/code\u003e) with all its variables. It was called by \u003ccode\u003eSubscriptionService.create_subscription\u003c/code\u003e, so there's a stack for that. Finally, the service was called by \u003ccode\u003eSubscriptionAPI.create_subscription\u003c/code\u003e so there's a frame for that one as well.\u003c/p\u003e\n\u003cp\u003eLet's look at how we'd achieve that.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eget_variables_from_all_calls\u003c/span\u003e() -\u0026gt; dict:\u003c/span\u003e\n \u003cspan class=\"hljs-string\"\u003e\"\"\"Get all the variables in the method that called us.\"\"\"\u003c/span\u003e\n local_variables = {}\n stack = inspect.stack()\n \u003cspan class=\"hljs-comment\"\u003e# We don't want the current frame, that's this function\u003c/span\u003e\n all_frames_except_this_one = stack[\u003cspan class=\"hljs-number\"\u003e1\u003c/span\u003e:]\n \u003cspan class=\"hljs-keyword\"\u003efor\u003c/span\u003e frame_information \u003cspan class=\"hljs-keyword\"\u003ein\u003c/span\u003e all_frames_except_this_one:\n frame = frame_information.frame\n local_variables.update(frame.f_locals)\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e local_variables\n\n\n\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionRepository\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self,\n ctx: Context,\n plan_id: UUID,\n user_id: UUID,\n business_id: UUID,\n status: SubscriptionStatus,\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Call the ORM to create the subscription\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Update the context with variables from all calls in the stack\u003c/span\u003e\n ctx.update(get_variables_from_all_calls()) \n \u003cspan class=\"hljs-comment\"\u003e# Serialize the output as a Subcription object\u003c/span\u003e\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e Subscription(...)\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eIn this example, I'm introducing a helper (\u003ccode\u003eget_variables_from_all_calls\u003c/code\u003e) to move the logic outside of our main function. Since the repository will call this method, that means Python will add a frame to the stack for this helper function as well. As such, we need to get all the frames except the current one. This explains why we're only capturing frames from index \u003ccode\u003e1\u003c/code"])</script><script>self.__next_f.push([1,"\u003e and beyond. Finally, in \u003ccode\u003eSubscriptionRepository.create_subscription\u003c/code\u003e we're calling \u003ccode\u003eget_variables_from_all_calls\u003c/code\u003e to get all the variables for us.\u003c/p\u003e\n\u003cp\u003eWhen we run the code, the \u003ccode\u003ectx\u003c/code\u003e has the following contents:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e{\u003cspan class=\"hljs-string\"\u003e'business_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'3498a3df-5001-481c-b0b8-a007609fb93c'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'plan_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'fefbbafd-0f59-49bb-9255-3c09eef38fec'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'status'\u003c/span\u003e: \u0026lt;SubscriptionStatus.ACTIVE: \u003cspan class=\"hljs-string\"\u003e'active'\u003c/span\u003e\u0026gt;,\n \u003cspan class=\"hljs-string\"\u003e'user_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'de19ca36-67c5-48eb-80c8-70b0db9575ac'\u003c/span\u003e)}\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eIf you're paying close attention, you'll notice this list is a little shorter than the last time we printed out the \u003ccode\u003eContext\u003c/code\u003e. In particular, this is missing the \u003ccode\u003eserialized_subscription\u003c/code\u003e and \u003ccode\u003esubscription\u003c/code\u003e object found in the \u003ccode\u003eSubscriptionAPI.create_subscription\u003c/code\u003e method. That happens because when we're capturing the variables, those don't exist yet.\u003c/p\u003e\n\u003cp\u003eSee below if you're not convinced.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-class\"\u003e\u003cspan class=\"hljs-keyword\"\u003eclass\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003eSubscriptionAPI\u003c/span\u003e:\u003c/span\u003e\n\n \u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003ecreate_subscription\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003e\n self, ctx: Context, plan_id: UUID, user_id: UUID, business_id: UUID\n \u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# The subscription variable is defined AFTER the call returns\u003c/span\u003e\n subscription = SubscriptionService().create_subscription(...)\n \u003cspan class=\"hljs-comment\"\u003e# Similarly, this is also defined AFTER the call returns\u003c/span\u003e\n serialized_subscription = {...}\n\n \u003cspan class=\"hljs-keyword\"\u003ereturn\u003c/span\u003e serialized_subscription\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eWe'll keep going! That's a small problem in the grand scheme of things. As you'll see in the next step... we've got bigger problems.\u003c/p\u003e\n\u003ch2 id=\"heading-step-3-dont-touch-the-code\"\u003eStep 3 - Don't touch the code\u003c/h2\u003e\n\u003cp\u003eOne of our primary goals was to enable telemetry without manual effort. If we need to be adding a bunch of \u003ccode\u003eget_variables_from_all_calls\u003c/code\u003e calls all over our code, that simply won't work. It's less work than manually adding every variable, but still too much effort. Additionally, we don't want folks to modify their code to include these calls. We want these utilities to be mostly invisible but providing all the benefits necessary.\u003c/p\u003e\n\u003cp\u003eLet's iterate and see what we can accomplish...\u003c/p\u003e\n\u003cp\u003eWhat if we tried capturing the stack variables after the we made the function call.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003esubscription_api = SubscriptionAPI()\nctx = {}\n\nsubscription_api.create_subscription(\n ctx, plan_id, user_id, business_id)\n\n\u003cspan class=\"hljs-comment\"\u003e# Capture all variables\u003c/span\u003e\nget_variables_from_all_calls()\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eIf we print out the context of the \u003ccode\u003eContext\u003c/code\u003e, we'd get the following:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e{}\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eThe \u003ccode\u003eContext\u003c/code\u003e is empty. Yeah, that's right... it's completely empty.\u003c/p\u003e\n\u003cp\u003eHere comes another important detail I omitted before...\u003c/p\u003e\n\u003cblockquote\u003e\n\u003cp\u003e\u003cstrong\u003eOnce a function is done executing, Python pops its frame from the stack.\u003c/strong\u003e\u003c/p\u003e\n\u003c/blockquote\u003e\n\u003cp\u003eSo... if you were to look at the stack at this point, it'd be empty (technically there's a \"module\" frame, but I've omitted that for brevity). There are no other frames on the stack, because after the function calls have exited, the frames associated with each call are completely gone. All of the previous approaches worked because we were capturing variables while we were \u003cem\u003ein\u003c/em\u003e the call stack. Once those functions have exited we're no longer in the call stack and don't have access to those variables.\u003c/p\u003e\n\u003cp\u003eSo we need a different approach. We need to c"])</script><script>self.__next_f.push([1,"apture variables while the relevant frames still exist on the stack. But how...\u003c/p\u003e\n\u003ch2 id=\"heading-step-4-byte-off-more-than-you-can-chew\"\u003eStep 4 - Byte off more than you can chew\u003c/h2\u003e\n\u003cp\u003eIt turns out, there's a way we can utilize the same approach, without modifying the code folks would write. We can introduce our capturing logic, at a lower-level... much lower. When we write Python code, it gets compiled into bytecode, which is then run by the interpreter.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1719277675826/ba7ff392-856d-4314-b83b-2cfa8ad75c03.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eWe could \u003cem\u003emodify\u003c/em\u003e the bytecode to accomplish the same objective. Python bytecode looks very different from Python. It's a sequence of instructions for the Python virtual machine. These instructions manipulate the internal stacks that Python uses. By modifying the resulting bytecode we can inject our logic and have the compiler execute our logic and capture the variables.\u003c/p\u003e\n\u003cp\u003eTo exactly replicate what we did at a higher level, we need to capture the stack just before the function returns. So, we'd need to parse the bytecode instructions, looking for a return statement and then inject our logic right before that. We'd insert our logic to call the \u003ccode\u003eget_variables_from_all_calls\u003c/code\u003e function which would extract the variables from the stack. Then finally we'd relace the function's bytecode with our patched version. Sounds simple enough, right?\u003c/p\u003e\n\u003cp\u003eThere are lots of different bytecode instructions. The following are the ones we're interested in:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003cp\u003e\u003ccode\u003eLOAD_GLOBAL\u003c/code\u003e - this instruction enables us to make our function available for calling. This instruction tells Python to look up an object in the global namespace and place it on the evaluation stack. We'll use this to make our function ready for calling.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003ccode\u003eLOAD_FAST\u003c/code\u003e - this instruction enables us to provide arguments for a function call. This pushes a variable unto the stack for evaluation.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003ccode\u003eCALL_FUNCTION\u003c/code\u003e - this instruction is used to execute our function call. With this instruction we can also provide a number of positional arguments that should be popped and passed to the function.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003ccode\u003ePOP_TOP\u003c/code\u003e - this instruction allows us to remove our function from the stack to continue the flow of the program. We'll use this to pop our function after we've called it.\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003eNext, I need to explain an idiosyncrasy we have to be familiar with to make this all work. Sometimes instructions spans multiple lines. We only need to add our patching instructions once. As such, we need to keep track of whether we've patched a line already so we don't do it multiple times.\u003c/p\u003e\n\u003cp\u003eArmed with that knowledge let's examine the function that does the patching.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e\u003cspan class=\"hljs-function\"\u003e\u003cspan class=\"hljs-keyword\"\u003edef\u003c/span\u003e \u003cspan class=\"hljs-title\"\u003epatch_function_to_capture_stack_variables\u003c/span\u003e(\u003cspan class=\"hljs-params\"\u003efunc\u003c/span\u003e):\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Get the bytecode that we plan to patch\u003c/span\u003e\n code = func.__code__\n bytecode = Bytecode.from_code(code)\n\n \u003cspan class=\"hljs-comment\"\u003e# Get a list of all line numbers for return instructions\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Recall instructions can span multiple lines\u003c/span\u003e\n return_line_numbers = [\n instruction.lineno\n \u003cspan class=\"hljs-keyword\"\u003efor\u003c/span\u003e instruction \u003cspan class=\"hljs-keyword\"\u003ein\u003c/span\u003e bytecode\n \u003cspan class=\"hljs-keyword\"\u003eif\u003c/span\u003e instruction.name == \u003cspan class=\"hljs-string\"\u003e\"RETURN_VALUE\"\u003c/span\u003e\n ]\n\n \u003cspan class=\"hljs-comment\"\u003e# For each return instruction, we want to insert a call to capture_stack\u003c/span\u003e\n \u003cspan class=\"hljs-keyword\"\u003efor\u003c/span\u003e return_line_number \u003cspan class=\"hljs-keyword\"\u003ein\u003c/span\u003e return_line_numbers:\n \u003cspan class=\"hljs-comment\"\u003e# construct the instructions we need\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# Add our function to the stack\u003c/span\u003e\n ad"])</script><script>self.__next_f.push([1,"d_capture_stack_function_to_the_stack = Instr(\u003cspan class=\"hljs-string\"\u003e\"LOAD_GLOBAL\"\u003c/span\u003e, \u003cspan class=\"hljs-string\"\u003e\"get_variables_from_all_calls\"\u003c/span\u003e)\n \u003cspan class=\"hljs-comment\"\u003e# Add the ctx as an argument for the function\u003c/span\u003e\n add_ctx_as_an_argument = Instr(\u003cspan class=\"hljs-string\"\u003e\"LOAD_FAST\"\u003c/span\u003e, \u003cspan class=\"hljs-string\"\u003e\"ctx\"\u003c/span\u003e)\n \u003cspan class=\"hljs-comment\"\u003e# Call our function and pop 1 arguemnt (ctx) to be passed to the function\u003c/span\u003e\n call_the_capture_stack_function = Instr(\u003cspan class=\"hljs-string\"\u003e\"CALL_FUNCTION\"\u003c/span\u003e, \u003cspan class=\"hljs-number\"\u003e1\u003c/span\u003e)\n \u003cspan class=\"hljs-comment\"\u003e# Remove our function to continue normal processing\u003c/span\u003e\n remove_the_capture_stack_function_from_the_stack = Instr(\u003cspan class=\"hljs-string\"\u003e\"POP_TOP\"\u003c/span\u003e)\n \u003cspan class=\"hljs-comment\"\u003e# Put the instructions all together\u003c/span\u003e\n catpure_stack_instructions = [\n add_capture_stack_function_to_the_stack,\n add_ctx_as_an_argument,\n call_the_capture_stack_function,\n remove_the_capture_stack_function_from_the_stack,\n ]\n\n \u003cspan class=\"hljs-keyword\"\u003efor\u003c/span\u003e i, instruction \u003cspan class=\"hljs-keyword\"\u003ein\u003c/span\u003e enumerate(bytecode):\n \u003cspan class=\"hljs-keyword\"\u003eif\u003c/span\u003e instruction.lineno == return_line_number:\n bytecode[i:i] = catpure_stack_instructions\n \u003cspan class=\"hljs-comment\"\u003e# There could be other instructions on the same line\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# We don't want to insert the new instructions multiple times\u003c/span\u003e\n \u003cspan class=\"hljs-comment\"\u003e# So break out of the loop\u003c/span\u003e\n \u003cspan class=\"hljs-keyword\"\u003ebreak\u003c/span\u003e\n\n updated_code = bytecode.to_code()\n\n func.__code__ = updated_code\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eAs shown in the code above, we're getting the bytecode, adding instructions to call our function and then applying the patch. We start by getting the \u003ccode\u003e__code__\u003c/code\u003e object of the function. That's the bytecode! We're using the \u003ccode\u003ebytecode\u003c/code\u003e library to simplify going to and from bytecode and construction of instructions. Next, we find all the lines that have return instructions. Finally, we construct our new instructions and add them to the bytecode.\u003c/p\u003e\n\u003cp\u003eFinally, we patch the \u003ccode\u003eSubscriptionRepository.create_subscription\u003c/code\u003e function so it'll call our function.\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003epatch_function_to_capture_stack_variables(\n SubscriptionRepository.create_subscription)\n\nsubscription_api = SubscriptionAPI()\nsubscription_api.create_subscription(\n ctx, plan_id, user_id, business_id)\n\nprint(ctx)\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eRunning this code, gets us the following output:\u003c/p\u003e\n\u003cpre\u003e\u003ccode class=\"lang-python\"\u003e {\u003cspan class=\"hljs-string\"\u003e'business_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'276dac6c-68ad-48bf-8a55-ac7ee6df64a5'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'plan_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'e72b5c80-63d7-4c5a-a2a9-d4b5eef937e0'\u003c/span\u003e),\n \u003cspan class=\"hljs-string\"\u003e'status'\u003c/span\u003e: \u0026lt;SubscriptionStatus.ACTIVE: \u003cspan class=\"hljs-string\"\u003e'active'\u003c/span\u003e\u0026gt;,\n \u003cspan class=\"hljs-string\"\u003e'user_id'\u003c/span\u003e: UUID(\u003cspan class=\"hljs-string\"\u003e'946bf774-e95a-488d-b5b5-55a255aa5d2a'\u003c/span\u003e)}\n\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eSuccess! Without asking anyone to modify their code, we were able to capture the local variables from the stack! This provides significant gains as we're able to reduce the manual effort involved. It's leaps and bounds further than where we started.\u003c/p\u003e\n\u003cp\u003eYou'll notice though that it's the same output as step 3. It still has the shortcoming where it doesn't capture variables that haven't been defined yet. There are ways to overcome this that don't take too much effort and don't re-introduce manual effort. Anyway... that's a story for another time.\u003c/p\u003e\n\u003ch2 id=\"heading-step-5-choke\"\u003eStep 5 - Choke\u003c/h2\u003e\n\u003cp\u003eI've had to acknowledge that despite the energy rush this investigation brought, its complexity is well beyond our company's maintenance capacity. None of us are ex"])</script><script>self.__next_f.push([1,"perts in the topic of bytecode. Theoretically, we're not doing much -- we're simply hooking into the code at a lower level. We wanted to avoid folks littering their code with our hooking logic, so we moved down an abstraction layer or two. That descent was unfortunately dangerously close to 6 feet under. Writing our Observability tooling hooks in bytecode would be suicidal for our tooling's life.\u003c/p\u003e\n\u003cp\u003eWhat's worse, is that the implementation I've described thus far is woefully inadequate. We'd need to make several improvements and additions to \u003cem\u003ebegin\u003c/em\u003e the journey to production. We'd need some more bytecode instructions to import our library from elsewhere (in the script it's defined in the same file). Discover and account for additional idiosyncrasies of bytecode. We haven't addressed the multiple ways that functions can exit (e.g. early returns, unexpected exceptions and try-catch scenarios). Additionally, we would want to make our code more robust. We don't want failures in our library to cascade to the rest of the application. We'd probably need to catch all failures in our calls and properly log them (e.g. Sentry, SumoLogic, DataDog, etc...), or capture with metrics. Then, finally, we haven't thought about optimization or security. Who knows what challenges lie in that space!\u003c/p\u003e\n\u003cp\u003eAnd so, we've started investigating alternatives. We've got some basic ideas we're toying around. One idea is to reduce the complexity, by only capturing function inputs and outputs. We could use a decorator-based approach to wrap public methods of all our components and capture the variables. We can implement our wrapping logic in a metaclass that we use to initialize our different components (e.g. services, repositories, clients, etc...). It's early days and we're still exploring things. I'll return to blog about our success (let's hope!) once we've found a working solution.\u003c/p\u003e\n\u003ch2 id=\"heading-conclusion\"\u003eConclusion\u003c/h2\u003e\n\u003cp\u003eIn our attempt to automate telemetry capture using Python, we explored the ambitious idea of introducing hooks at the bytecode level to automate this process. While our manual approach of enriching telemetry within our services has proven effective in reducing incident investigation time, scaling this across multiple teams and microservices was impractical. Our journey involved understanding Python's stack and bytecode intricacies to capture local variables across function calls. Despite the surge and adrenaline of this approach, the unavoidable complexity and maintenance challenges, lead us to abandon this method. We're now considering simpler, more maintainable alternatives like using decorators to capture function inputs and outputs.\u003c/p\u003e\n\u003cp\u003eThanks for reading! If you've got thoughts on this topic, I'd love to hear it! Feel free to reach out to me on \u003ca target=\"_blank\" href=\"https://x.com/kramnaej\" rel=\"noopener noreferrer nofollow ugc\"\u003eTwitter\u003c/a\u003e or \u003ca target=\"_blank\" href=\"https://www.linkedin.com/in/jean-mark-wright/\" rel=\"noopener noreferrer nofollow ugc\"\u003eLinkedIn\u003c/a\u003e!\u003c/p\u003e\n\u003cp\u003eAlso, many thanks to the \u003ca target=\"_blank\" href=\"https://github.com/DataDog/datadogpy\" rel=\"noopener noreferrer nofollow ugc\"\u003eDataDog library\u003c/a\u003e I reverse-engineered to get an idea of how this could be accomplished.\u003c/p\u003e\n\u003cp\u003eHere are a few resources that were helpful, if you're looking to learn more about bytecode.\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003cp\u003e\u003ca target=\"_blank\" href=\"https://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1\" rel=\"noopener noreferrer nofollow ugc\"\u003eDemystifying Python Bytecode: A Guide to Understanding and Analyzing Code Execution\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\u003cp\u003e\u003ca target=\"_blank\" href=\"https://github.com/DataDog/datadogpy\" rel=\"noopener noreferrer nofollow ugc\"\u003eDataDog Python Library\u003c/a\u003e\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003e\u003ca target=\"_blank\" href=\"https://medium.com/@noransaber685/demystifying-python-bytecode-a-guide-to-understanding-and-analyzing-code-execution-6a163cb83bd1\" rel=\"noopener noreferrer nofollow ugc\"\u003ehttps://medium.com/@noransaber685/demystifying-python-bytecode-a-gui"])</script><script>self.__next_f.push([1,"de-to-understanding-and-analyzing-code-execution-6a163cb83bd1\u003c/a\u003e\u003c/p\u003e\n25:[\"$\",\"div\",null,{\"className\":\"relative\",\"children\":[[\"$\",\"aside\",null,{\"className\":\"hidden xl:block absolute top-0 left-0 h-full\",\"children\":[\"$\",\"nav\",null,{\"className\":\"sticky top-22 w-44\",\"children\":[\"$\",\"$L2a\",null,{\"headings\":[{\"id\":\"heading-introduction\",\"text\":\"Introduction\",\"level\":2},{\"id\":\"heading-motivation\",\"text\":\"Motivation\",\"level\":2},{\"id\":\"heading-step-0-the-existing-manual-process\",\"text\":\"Step 0 - The existing, manual process\",\"level\":2},{\"id\":\"heading-step-1-capture-variables-in-a-method\",\"text\":\"Step 1 - Capture variables in a method\",\"level\":2},{\"id\":\"heading-step-2-capture-the-locals-from-functions-we-call\",\"text\":\"Step 2 - Capture the locals from functions we call\",\"level\":2},{\"id\":\"heading-step-3-dont-touch-the-code\",\"text\":\"Step 3 - Don't touch the code\",\"level\":2},{\"id\":\"heading-step-4-byte-off-more-than-you-can-chew\",\"text\":\"Step 4 - Byte off more than you can chew\",\"level\":2},{\"id\":\"heading-step-5-choke\",\"text\":\"Step 5 - Choke\",\"level\":2},{\"id\":\"heading-conclusion\",\"text\":\"Conclusion\",\"level\":2}]}]}]}],[\"$\",\"div\",null,{\"className\":\"max-w-2xl mx-auto min-w-0 contain-[inline-size]\",\"children\":[\"$\",\"div\",null,{\"className\":\"prose dark:prose-invert prose-pre:bg-foreground prose-pre:text-background prose-pre:border prose-pre:border-border prose-pre:rounded prose-pre:px-5 prose-pre:py-4 dark:prose-pre:bg-muted dark:prose-pre:text-foreground dark:prose-pre:border-border prose-pre:overflow-x-auto max-w-none text-base sm:text-lg [\u0026\u003ediv\u003ep:first-child]:mt-0 [\u0026\u003ediv\u003ep:first-child]:pt-0 min-w-0 wrap-break-word [\u0026_a]:break-all **:max-w-full\",\"children\":[[],[\"$\",\"div\",null,{\"suppressHydrationWarning\":true,\"dangerouslySetInnerHTML\":{\"__html\":\"$2b\"}}],[],\"$L2c\",\"$L2d\",\"$L2e\",\"$L2f\",\"$L30\"]}]}]]}]\n26:[\"$\",\"div\",null,{\"className\":\"max-w-2xl mx-auto border-t border-border pt-8 space-y-5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center gap-2\",\"children\":[[\"$\",\"$L21\",\"56744721958ef13879b94d67\",{\"href\":\"/tag/python\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"python\"]}],[\"$\",\"$L21\",\"5c513528a8f986a7229128d9\",{\"href\":\"/tag/bytecode\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"bytecode\"]}],[\"$\",\"$L21\",\"5f1df3b4f4016901885ccc96\",{\"href\":\"/tag/observability\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"observability\"]}],[\"$\",\"$L21\",\"632ae84c0b80d2f3c43f152b\",{\"href\":\"/tag/telemetry\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"telemetry\"]}],[\"$\",\"$L21\",\"56744721958ef13879b948a2\",{\"href\":\"/tag/microservices\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"microservices\"]}],[\"$\",\"$L21\",\"5fc2287b55b779631e1acc29\",{\"href\":\"/tag/instrumentation\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"instrumentation\"]}]]}],false]}]\n31:I[32478,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191"])</script><script>self.__next_f.push([1,"af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"CodeHighlighter\"]\n32:I[7714,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"EmbedHandler\"]\n33:I[54960,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"ImageZoom\"]\n34:I[6671,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"MathJaxHandler\"]\n35:I[91337,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"MermaidHandler\"]\n2c:[\"$\",\"$L31\",null,{}]\n2d:[\"$\",\"$L32\",null,{}]\n2e:[\"$\",\"$L33\",null,{}]\n2f:[\"$\",\"$L34\",null,{\"hasLatex\":false}]\n30:[\"$\",\"$L35\",null,{}]\n36:Tc98,M268.379 39q-6.322 0-10.35-4.108-4.002-4.134-4.002-10.483 0-6.296 4.002-10.43 4-4.135 10.163-4.135 6.028 0 9.817 3.734 3.814 3.735 3.814 9.497v2.8h-24.248v-4.054h17.633q-.187-2.747-2.054-4.508-1.868-1.76-4.962-1.76-3.28-.001-5.388 2.374-2.108 2.373-2.108 6.375 0 4.188 2.348 6.695 2.347 2.481 5.895 2.481 1.68 0 2.934-.453a8.5 8.5 0 0 0 2.321-1.307q1.041-.854 2.268-2.614l4.694 2.96q-1.707 2.748-3.494 4.135a12.7 12.7 0 0 1-3.975 2.081q-2.187.72-5.308.72m-33.496 0q-5.575 0-9.283-4.028-3.681-4.029-3.681-10.484 0-6.43 3.761-10.51 3.76-4.108 9.47-4.108 2.8 0 5.175 1.2 2.4 1.174 3.628 3.148l.026-.08V0h7.016v38.2h-6.722v-4.722l.64.8-.694.133q-1.28 2.108-3.788 3.361Q237.924 39 234.883 39m1.627-6.002q3.308 0 5.495-2.428t2.188-6.135q0-3.762-2.214-6.109-2.214-2.374-5.469-2.374-3.28 0-5.522 2.347-2.214 2.348-2.214 6.163 0 3.76 2.214 6.162 2.214 2.374 5.522 2.374M205.811 39q-6.376 0-10.537-4.161-4.161-4.188-4.161-10.404 0-6.321 4.161-10.456 4.188-4.135 10.537-4.135 6.348 0 10.537 4.161 4.188 4.161 4.188 10.43 0 6.216-4.188 10.404Q212.186 39 205.811 39m0-6.162q3.307 0 5.575-2.374 2.267-2.374 2.267-6.055 0-3.707-2.294-6.056-2.267-2.347-5.548-2.347-3.308 0-5.575 2.374-2.268 2.348-2.268 6.029t2.241 6.055q2.268 2.374 5.602 2.374M162.586 38.2V10.644h6.802v5.015l-.8-.987h.906q1.308-2.187 3.762-3.494 2.48-1.335 5.228-1.334 4.615 0 7.256 2.8 2.667 2.776 2.667 7.603V38.2h-7.069V21.794q0-2.854-1.387-4.401t-4.134-1.547q-2.614 0-4.402 1.974-1.76 1.974-1.76 4.881V38.2zm-30.181 0V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255"])</script><script>self.__next_f.push([1," 2.828 2.668 2.801 2.668 7.522V38.2h-7.069V21.794q0-2.827-1.387-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.814 2.027-1.814 4.962v15.391zm-14.416.8q-2.882 0-5.015-.64-2.108-.614-3.761-1.84-1.654-1.228-2.668-2.695l-.987-1.467 4.695-3.174.773 1.067a9.1 9.1 0 0 0 1.788 1.814 8.2 8.2 0 0 0 2.32 1.2q1.308.4 3.068.4 2.161 0 3.441-.8 1.308-.8 1.307-2.294 0-1.36-1.147-2.108-1.12-.747-5.068-1.654-5.335-1.253-7.656-3.334-2.32-2.107-2.321-5.362 0-3.76 3.148-6.002 3.148-2.267 7.923-2.267 2.454 0 4.348.507t3.388 1.493q1.493.988 2.454 2.188l.987 1.2-4.482 3.255-.667-.774a7.4 7.4 0 0 0-1.627-1.387 6.4 6.4 0 0 0-2.054-.88 9.5 9.5 0 0 0-2.401-.294q-1.974 0-3.201.8-1.2.774-1.2 1.921 0 1.254 1.12 2.027 1.147.748 5.229 1.681 5.334 1.227 7.629 3.308 2.294 2.055 2.294 5.308 0 4.029-3.228 6.43Q123.217 39 117.989 39M87.61 39q-4.508 0-7.335-2.454-2.828-2.481-2.828-6.482 0-4.188 3.254-6.669t8.483-2.48q2.082 0 4.028.426 1.974.427 3.361 1.12v-1.6q0-2.588-1.813-4.082-1.788-1.493-4.615-1.493-1.628 0-2.801.373a8.7 8.7 0 0 0-2.188.987q-.987.614-1.84 1.414l-.854.8-4.081-3.788 1.307-1.173a18.7 18.7 0 0 1 2.934-2.108q1.601-.933 3.601-1.44t4.589-.507q5.868 0 9.229 2.88 3.361 2.882 3.361 8.084V38.2h-6.749v-5.922l1.654 2.187h-1.76q-1.308 2.055-3.655 3.308Q90.545 39 87.611 39m1.814-4.988q3.148 0 5.149-2.187 2-2.215 2-5.309v-.08q-1.12-.693-2.72-1.094a13 13 0 0 0-3.388-.426q-2.828 0-4.562 1.2-1.734 1.173-1.734 3.521 0 2.028 1.468 3.201t3.787 1.174M49 38.2V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.07V21.794q0-2.827-1.386-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.815 2.027-1.814 4.962v15.391z1e:[\"$\",\"footer\",null,{\"className\":\"px-4 pt-8 pb-20 sm:px-8 text-muted-foreground\",\"aria-label\":\"Footer\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-4xl mx-auto\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex flex-col items-center gap-4 text-center text-sm\",\"children\":[false,[\"$\",\"p\",null,{\"className\":\"font-medium\",\"children\":[[\"$\",\"time\",null,{\"dateTime\":\"2026\",\"children\":[[\"$\",\"span\",null,{\"className\":\"font-sans\",\"children\":\"©\"}],\" \",2026]}],\" \",\"Perspective Unspoken\"]}],[\"$\",\"nav\",null,{\"aria-label\":\"Footer navigation\",\"children\":[\"$\",\"ul\",null,{\"className\":\"flex flex-wrap justify-center gap-4 text-sm tracking-wide\",\"children\":[[\"$\",\"li\",\"/archive\",{\"children\":[\"$\",\"$L21\",null,{\"href\":\"/archive\",\"className\":\"transition hover:text-foreground\",\"target\":\"$undefined\",\"rel\":\"$undefined\",\"children\":\"Archive\"}]}],[\"$\",\"li\",\"https://hashnode.com/privacy-policy\",{\"children\":[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/privacy-policy\",\"className\":\"transition hover:text-foreground\",\"target\":\"_blank\",\"rel\":\"noopener\",\"children\":\"Privacy\"}]}],[\"$\",\"li\",\"https://hashnode.com/terms\",{\"children\":[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/terms\",\"className\":\"transition hover:text-foreground\",\"target\":\"_blank\",\"rel\":\"noopener\",\"children\":\"Terms\"}]}]]}]}],[\"$\",\"div\",null,{\"className\":\"pt-8 flex justify-center gap-6\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"/sitemap.xml\",\"target\":\"_blank\",\"rel\":\"noopener\",\"className\":\"flex items-center gap-2 text-sm transition hover:text-foreground\",\"title\":\"Sitemap\",\"children\":[[\"$\",\"i\",null,{\"className\":\"fa-regular fa-sitemap\"}],[\"$\",\"span\",null,{\"children\":\"Sitemap\"}]]}],[\"$\",\"$L21\",null,{\"href\":\"/rss.xml\",\"target\":\"_blank\",\"rel\":\"noopener\",\"className\":\"flex items-center gap-2 text-sm transition hover:text-foreground\",\"title\":\"RSS Feed\",\"children\":[[\"$\",\"i\",null,{\"className\":\"fa-regular fa-rss\"}],[\"$\",\"span\",null,{\"children\":\"RSS\"}]]}]]}],[\"$\",\"div\",null,{\"className\":\"pt-10 text-center\",\"children\":[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev\u0026utm_medium=referral\u0026utm_campaign=blog_footer_logo\u0026utm_content=logo\",\"target\":\"_blank\",\"rel\":\"noopener\",\"aria-label\":\"Visit Hashnode (opens in a new tab)\",\"children\":[\"$\",\"svg\",null,{\"fill\":\"none\",\"viewBox\":\"0 0 282 39\",\"aria-hidden\":\"true\",\"className\":\"h-6 w-auto min-w-[9.5rem] text-muted-foreground transition hover:text-foreground\",\"children\":[[\"$\",\"path\",null,{\"f"])</script><script>self.__next_f.push([1,"ill\":\"currentColor\",\"d\":\"M2.68 13.032c-3.573 3.505-3.573 9.363 0 12.936L13.032 36.32c3.505 3.573 9.363 3.573 12.936 0L36.32 25.968c3.573-3.573 3.573-9.431 0-12.936L25.968 2.68c-3.573-3.573-9.431-3.573-12.936 0zm12.211 1.935c2.507-2.521 6.582-2.544 9.104-.038s2.544 6.582.038 9.104-6.582 2.544-9.104.038-2.544-6.582-.038-9.104\"}],[\"$\",\"path\",null,{\"fill\":\"currentColor\",\"d\":\"$36\"}]]}],\"className\":\"inline-flex items-center justify-center gap-2 whitespace-nowrap font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [\u0026_svg]:pointer-events-none [\u0026_svg]:size-4 [\u0026_svg]:shrink-0 hover:bg-accent hover:text-accent-foreground h-10 rounded-md text-xl px-5 py-4\",\"ref\":null}]}]]}]}]}]\n"])</script></body></html>