56 lines
150 KiB
HTML
56 lines
150 KiB
HTML
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="image" href="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg"/><link rel="stylesheet" href="/_next/static/css/bc5c0eaecd39bc7d.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/7ffcc02a3c67389b.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/c20e76e7d11dec32.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/a6eb8c1f37be68d5.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/webpack-d716a75e4017380e.js"/><script src="/_next/static/chunks/4bd1b696-100b9d70ed4e49c1.js" async=""></script><script src="/_next/static/chunks/9da6db1e-8a4e8c23641c31bf.js" async=""></script><script src="/_next/static/chunks/1255-a242e0ce0372fddf.js" async=""></script><script src="/_next/static/chunks/main-app-bda8a52fe841b3aa.js" async=""></script><script src="/_next/static/chunks/4554-984b480a9db5f3fe.js" async=""></script><script src="/_next/static/chunks/7563-4dfa0994a9b0e37c.js" async=""></script><script src="/_next/static/chunks/9216-9272709ba40c1cfd.js" async=""></script><script src="/_next/static/chunks/3874-323555cd5c106ba4.js" async=""></script><script src="/_next/static/chunks/3091-bf1537ff6571a1bd.js" async=""></script><script src="/_next/static/chunks/app/layout-f61a55efa656d91f.js" async=""></script><script src="/_next/static/chunks/app/not-found-b861935655c5698b.js" async=""></script><script src="/_next/static/chunks/5001-fae0bebdd93f95e3.js" async=""></script><script src="/_next/static/chunks/2423-c8b1388240338d7d.js" async=""></script><script src="/_next/static/chunks/652-f40dcb480b4e356f.js" async=""></script><script src="/_next/static/chunks/730-8191af9e9424b148.js" async=""></script><script src="/_next/static/chunks/app/page-cc18ae1a3a7a5ed8.js" async=""></script><script src="/_next/static/chunks/0dbeb660-9e82e99813552576.js" async=""></script><script src="/_next/static/chunks/1545-7abf96433ff6553f.js" async=""></script><script src="/_next/static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js" async=""></script><meta name="next-size-adjust" content=""/><title>Cascading failures and the impossibility of scheduling team lunches</title><meta name="description" content="Introduction
|
||
In this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't..."/><link rel="canonical" href="https://jaywhy13.hashnode.dev/cascading-failures-and-the-impossibility-of-scheduling-team-lunches"/><meta property="og:title" content="Cascading failures and the impossibility of scheduling team lunches"/><meta property="og:description" content="Introduction
|
||
In this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't..."/><meta property="og:url" content="https://jaywhy13.hashnode.dev/cascading-failures-and-the-impossibility-of-scheduling-team-lunches"/><meta property="og:site_name" content="Perspective Unspoken"/><meta property="og:image" content="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg"/><meta property="og:image:width" content="1200"/><meta property="og:image:height" content="630"/><meta property="og:type" content="article"/><meta property="article:published_time" content="2023-02-09T12:15:47.579Z"/><meta property="article:author" content="https://jaywhy13.hashnode.dev/@jaywhy13"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:creator" content="https://twitter.com/kramnaej"/><meta name="twitter:title" content="Cascading failures and the impossibility of scheduling team lunches"/><meta name="twitter:description" content="Introduction
|
||
In this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't..."/><meta name="twitter:image" content="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg"/><link rel="icon" href="/favicon.ico"/><script type="text/javascript">window.addEventListener('message',function(e){var d=e.data;if(!d||d.__hnWidget!==1||typeof d.h!=='number'||d.h<=0)return;var f=document.querySelectorAll('.hn-widget-frame');for(var i=0;i<f.length;i++){if(f[i].contentWindow===e.source){f[i].style.height=d.h+'px';break;}}});</script><script src="/_next/static/chunks/polyfills-42372ed130431b0a.js" noModule=""></script></head><body class="__variable_ede2c9 __variable_558dcf bg-background dark:bg-background"><div hidden=""><!--$--><!--/$--></div><script>((a2, b2, c4, d3, e2, f2, g4, h3) => {
|
||
let i3 = document.documentElement, j2 = ["light", "dark"];
|
||
function k2(b3) {
|
||
var c5;
|
||
(Array.isArray(a2) ? a2 : [a2]).forEach((a3) => {
|
||
let c6 = a3 === "class", d4 = c6 && f2 ? e2.map((a4) => f2[a4] || a4) : e2;
|
||
c6 ? (i3.classList.remove(...d4), i3.classList.add(f2 && f2[b3] ? f2[b3] : b3)) : i3.setAttribute(a3, b3);
|
||
}), c5 = b3, h3 && j2.includes(c5) && (i3.style.colorScheme = c5);
|
||
}
|
||
if (d3) k2(d3);
|
||
else try {
|
||
let a3 = localStorage.getItem(b2) || c4, d4 = g4 && a3 === "system" ? window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light" : a3;
|
||
k2(d4);
|
||
} catch {
|
||
}
|
||
})("class","theme","system",null,["light","dark"],null,true,true)</script><main id="main-content" class="min-h-screen bg-transparent"><header class="sticky top-0 z-50 w-full py-4 bg-background backdrop-blur-sm transition-transform duration-300 translate-y-0"><a href="#main-content" class="sr-only focus:not-sr-only focus:absolute focus:left-4 focus:top-4 focus:z-50 focus:rounded-md focus:bg-background focus:px-4 focus:py-2 focus:text-sm focus:shadow">Skip to main content</a><div class="relative w-full mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem] px-4 sm:px-12 md:px-18 lg:px-16 xl:px-10"><div class="flex items-center justify-between gap-3 xl:hidden"><div class="flex items-center gap-3 min-w-0 flex-1"><a href="https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev&utm_medium=referral&utm_campaign=blog_header_logo&utm_content=logo" class="inline-flex items-center justify-center text-lg text-muted-foreground hover:text-foreground transition border rounded-md h-9 w-9 shrink-0 border-border hover:bg-muted"><i class="text-xl fa-brands fa-hashnode" aria-hidden="true"></i><span class="sr-only">Hashnode</span></a><a title="Perspective Unspoken" class="text-lg font-semibold text-foreground truncate min-w-0" href="/">Perspective Unspoken</a></div><div class="flex items-center gap-2 shrink-0"><button class="gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 hover:bg-accent py-2 h-9 w-9 px-0 flex items-center justify-center text-muted-foreground hover:text-foreground bg-transparent border-border shadow-none" aria-label="Open search (press Control or Command and K)"><i class="fa-jelly fa-search text-sm" aria-hidden="true"></i><span class="sr-only">Open search (press Control or Command and K)</span></button><button class="inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 hover:bg-accent w-9 px-3 h-9 text-muted-foreground hover:text-foreground shadow-none" type="button" id="radix-_R_2j4anpfalb_" aria-haspopup="menu" aria-expanded="false" data-state="closed" data-slot="dropdown-menu-trigger"><i class="fa-jelly fa-sun text-sm scale-100 rotate-0 transition-all dark:scale-0 dark:-rotate-90" aria-hidden="true"></i><i class="fa-jelly fa-moon absolute text-sm scale-0 rotate-90 transition-all dark:scale-100 dark:rotate-0" aria-hidden="true"></i><span class="sr-only">Toggle theme</span></button><button class="inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 border border-input bg-background hover:bg-accent hover:text-accent-foreground py-2 h-9 w-10 px-0 shadow-none" type="button" aria-haspopup="dialog" aria-expanded="false" aria-controls="radix-_R_3j4anpfalb_" data-state="closed" data-slot="sheet-trigger"><i class="fa-jelly fa-bars" aria-hidden="true"></i><span class="sr-only">Open menu</span></button></div></div><div class="hidden xl:grid gap-4 items-center xl:grid-cols-[1fr_auto_1fr] xl:gap-6 max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="flex items-center w-full gap-3 min-w-0"><a href="https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev&utm_medium=referral&utm_campaign=blog_header_logo&utm_content=logo" class="inline-flex items-center justify-center text-lg text-muted-foreground hover:text-foreground transition border rounded-md h-9 w-9 border-border hover:bg-muted"><i class="text-xl fa-brands fa-hashnode" aria-hidden="true"></i><span class="sr-only">Hashnode</span></a><span class="h-6 border-l border-border"></span><p class="font-semibold text-foreground min-w-0 truncate text-lg sm:text-xl"><a title="Perspective Unspoken" href="/">Perspective Unspoken</a></p></div><nav aria-label="Main navigation" class="flex justify-center w-full"><nav aria-label="Main" data-orientation="horizontal" dir="ltr" data-slot="navigation-menu" data-viewport="false" class="group/navigation-menu relative flex flex-1 items-center max-w-max justify-center"><div style="position:relative"><ul data-orientation="horizontal" data-slot="navigation-menu-list" class="group flex flex-1 list-none items-center flex-nowrap justify-center gap-1 text-muted-foreground text-base" dir="ltr"><li data-slot="navigation-menu-item" class="relative"><a target="_blank" rel="noopener" data-slot="navigation-menu-link" class="data-[active=true]:focus:bg-accent data-[active=true]:hover:bg-accent data-[active=true]:bg-accent/50 data-[active=true]:text-accent-foreground focus:bg-accent focus:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground flex-col p-2 outline-none focus-visible:outline-1 [&_svg:not([class*='size-'])]:size-4 inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 hover:bg-accent hover:text-accent-foreground h-9 px-4 py-2" data-radix-collection-item="" href="https://calendar.google.com/calendar/u/0/appointments/AcZssZ3tWQd0YSITufTZaAitKeE6Ln2NBOouN5Qlop4=?gv=true">Calendar</a></li></ul></div></nav></nav><div class="flex items-center justify-end gap-3 text-muted-foreground"><button class="gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 hover:bg-accent py-2 h-9 w-9 px-0 flex items-center justify-center text-muted-foreground hover:text-foreground bg-transparent border-border shadow-none" aria-label="Open search (press Control or Command and K)"><i class="fa-jelly fa-search text-sm" aria-hidden="true"></i><span class="sr-only">Open search (press Control or Command and K)</span></button><div class="flex items-center gap-3"><button class="inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 hover:bg-accent w-9 px-3 h-9 text-muted-foreground hover:text-foreground shadow-none" type="button" id="radix-_R_6t4anpfalb_" aria-haspopup="menu" aria-expanded="false" data-state="closed" data-slot="dropdown-menu-trigger"><i class="fa-jelly fa-sun text-sm scale-100 rotate-0 transition-all dark:scale-0 dark:-rotate-90" aria-hidden="true"></i><i class="fa-jelly fa-moon absolute text-sm scale-0 rotate-90 transition-all dark:scale-100 dark:rotate-0" aria-hidden="true"></i><span class="sr-only">Toggle theme</span></button><a target="_blank" rel="noopener" class="inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 border border-input bg-background hover:bg-accent hover:text-accent-foreground py-2 gap-2 px-3 h-9 shadow-none" href="https://hashnode.com"><i class="text-sm fa-jelly fa-pencil" aria-hidden="true"></i>Write</a></div></div></div></div></header><div data-slot="dialog-header" class="flex flex-col gap-2 text-center sm:text-left sr-only"><h2 id="radix-_R_8anpfalbH1_" data-slot="dialog-title" class="text-lg leading-none font-semibold">Command Palette</h2><p id="radix-_R_8anpfalbH2_" data-slot="dialog-description" class="text-muted-foreground text-sm">Search for a command to run...</p></div><section class="py-8"><div class="px-4 sm:px-8"><div class="w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="grid gap-5 max-w-full"><article class="w-full max-w-full rounded-xl border bg-card text-card-foreground"><div class="px-5 pt-12 pb-8 sm:px-8"><header class="space-y-5"><div class="space-y-4 text-center max-w-4xl mx-auto"><h1 class="text-2xl sm:text-3xl md:text-4xl lg:text-5xl font-bold leading-tight tracking-tight text-balance text-foreground">Cascading failures and the impossibility of scheduling team lunches</h1><p class="text-lg sm:text-xl leading-normal font-normal text-muted-foreground text-balance">An illustrated guide on the importance of implementing timeouts and other features to increase system reliability</p><div class="flex flex-wrap items-center justify-center gap-3 text-sm text-muted-foreground"><div class="flex items-center gap-2"><span>Published</span><time dateTime="2023-02-09T12:15:47.579Z">February 9, 2023</time></div><span class="text-muted-foreground">•</span><span>9<!-- --> min read</span><span class="text-muted-foreground">•</span><a href="/cascading-failures-and-the-impossibility-of-scheduling-team-lunches.md" target="_blank" rel="noopener" class="inline-flex items-center gap-2 text-muted-foreground transition-colors hover:text-foreground"><i class="fa-brands fa-markdown" aria-hidden="true"></i>View as Markdown</a></div></div></header></div><div class="w-full"><img alt="Cascading failures and the impossibility of scheduling team lunches" width="1200" height="750" decoding="async" data-nimg="1" class="w-full h-auto" style="color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg"/></div><div class="px-5 py-12 sm:px-8 space-y-8"><div class="max-w-2xl mx-auto"><div class="flex flex-wrap items-center gap-5"><div class="flex items-center gap-5"><div class="flex gap-3 items-start"><a target="_blank" rel="noopener" href="https://hashnode.com/@jaywhy13"><span class="relative flex shrink-0 overflow-hidden rounded-full h-10 w-10 border border-border transition-opacity hover:opacity-80"><span class="flex h-full w-full items-center justify-center rounded-full bg-muted text-muted-foreground font-semibold text-lg">J</span></span></a><div class="min-w-0"><div class="flex items-center gap-2"><a target="_blank" rel="noopener" class="font-medium text-foreground hover:text-primary transition-colors" href="https://hashnode.com/@jaywhy13">Jean-Mark Wright</a><div class="flex items-center gap-1.5 text-sm"><a target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground hover:text-foreground transition-colors" aria-label="X (Twitter)" href="https://twitter.com/kramnaej"><i class="fa-brands fa-x-twitter" aria-hidden="true"></i></a><a target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground hover:text-foreground transition-colors" aria-label="LinkedIn" href="https://www.linkedin.com/in/jean-mark-wright/"><i class="fa-brands fa-linkedin" aria-hidden="true"></i></a></div></div><div class="text-sm text-muted-foreground line-clamp-2"><p>Jean-Mark is an articulate, customer-focused, visionary with extensive industry experience encompassing front and back end development in all phases of software development. He lives "outside the box" in pursuit of architectural elegance and relevant, tangible solutions. He is a strong believer that team empowerment fosters genius and is versed at working solo.</p>
|
||
</div></div></div></div></div></div><div class="relative"><aside class="hidden xl:block absolute top-0 left-0 h-full"><nav class="sticky top-22 w-44"><div class="hidden xl:block group/toc relative"><div class="grid"><div class="[grid-area:1/1] opacity-0 group-hover/toc:opacity-100 transition-opacity duration-200 max-h-[calc(100vh-3rem)] overflow-y-auto"><p class="text-sm font-semibold uppercase tracking-wide text-muted-foreground mb-3">On this page</p><nav aria-label="On this page"><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-introduction">Introduction</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-its-lunchtime">It's lunchtime!</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-there-must-be-a-better-way">There must be a better way</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0.75rem" href="#heading-issue-a-deadline-timeouts-retries-backoff-and-circuit-breakers">Issue a deadline - timeouts, retries, backoff, and circuit breakers</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:1.5rem" href="#heading-timeouts">Timeouts</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:1.5rem" href="#heading-retries-and-backoff">Retries and Backoff</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:1.5rem" href="#heading-circuit-breakers">Circuit Breakers</a><a class="block py-1 text-sm transition-colors hover:text-foreground text-muted-foreground" style="padding-left:0rem" href="#heading-conclusion">Conclusion</a></nav></div><div class="[grid-area:1/1] relative flex flex-col gap-px transition-opacity duration-200 group-hover/toc:opacity-0 pointer-events-none self-start max-h-[calc(100vh-3rem)] overflow-hidden pl-3"><div class="absolute left-0 w-0.5 rounded-full bg-foreground transition-all duration-300 opacity-0" style="transform:translateY(0px);height:12px"></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:12px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:15px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:54px"></div></div><div class="flex items-center h-3" style="padding-left:10px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:48px"></div></div><div class="flex items-center h-3" style="padding-left:20px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:10px"></div></div><div class="flex items-center h-3" style="padding-left:20px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:19px"></div></div><div class="flex items-center h-3" style="padding-left:20px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:16px"></div></div><div class="flex items-center h-3" style="padding-left:0px"><div class="h-0.5 rounded-full transition-all duration-300 bg-border" style="width:10px"></div></div></div></div></div></nav></aside><div class="max-w-2xl mx-auto min-w-0 contain-[inline-size]"><div class="prose dark:prose-invert prose-pre:bg-foreground prose-pre:text-background prose-pre:border prose-pre:border-border prose-pre:rounded prose-pre:px-5 prose-pre:py-4 dark:prose-pre:bg-muted dark:prose-pre:text-foreground dark:prose-pre:border-border prose-pre:overflow-x-auto max-w-none text-base sm:text-lg [&>div>p:first-child]:mt-0 [&>div>p:first-child]:pt-0 min-w-0 wrap-break-word [&_a]:break-all **:max-w-full"><div><h2 id="heading-introduction">Introduction</h2>
|
||
<p>In this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't want that failure propagating, or cascading to other areas. You must control the blast radius, or everything will go up in flames. Ideally, you want to ensure that your components absorb shocks and strains and don't provide a pathway for propagation. One uncontained fault can jeopardize your system's availability and reliability. We can use timeouts, circuit breakers, and other techniques to reduce the impact.</p>
|
||
<h2 id="heading-its-lunchtime">It's lunchtime!</h2>
|
||
<p>To illustrate the problem, we'll consider the task of scheduling team lunch for a rather busy set of colleagues. Let's assume everybody is working remotely and tends to have a life, which gets in the way of team initiatives. This makes scheduling a rather interesting adventure. Nobody pays attention to the polls in Slack to decide on a time for lunch, so we have to resort to calling everyone manually. For our consideration, let's assume I called the restaurant and was fortunate enough to get a patient representative who waited while I called my other team members to book.</p>
|
||
<p>Have a look at the diagram below. Hopefully, it looks familiar. It's a trace graph! Each box represents some action that's happening. The first box represents the duration of my call to the restaurant. The boxes in the next line represent each co-worker I'm calling in an attempt to schedule the team lunch. Each box below represents the life commitments that make scheduling a time-consuming affair. Let's take a closer look!</p>
|
||
<p><img src="https://cdn.hashnode.com/res/hashnode/image/upload/v1674532131705/8577af52-25a5-4e67-aaec-029cd19d8ee7.png" alt class="image--center mx-auto" /></p>
|
||
<p>So I called the restaurant and asked them to hold while I connected with my colleagues and confirmed. First up, I got a Pakistani colleague we'll call Nima. He has a beautiful baby girl. When I called he just happened to be in the middle of a diaper change. Yikes. After surgically removing the diaper and installing a clean one, it took him a few more minutes to get back to his computer and check his calendar. He's free! Next up, Kan, a health-conscious teammate who faithfully and consistently goes to the gym once or twice every couple of months. Just my luck! Gym time just happens to be now. I had to wait until he signed in and did his intensive workout session consisting of inhaling and stretching. 15 minutes in, he's able to get back to his computer and check his schedule, he's free! Finally, let's check in with Julie. It's almost 20 minutes and I can hear the receptionist on the line getting impatient. She's having issues getting Zoom started, and she's about to head into a quick meeting. Close to the half-hour mark, Julie confirms that she too is free! Overjoyed, the receptionist books us in and terminates the call.</p>
|
||
<p>Phew... that only took 30 minutes. The receptionist had to wait for me to connect with all my colleagues. All of them had different things on their plate and were unable to respond immediately. In many cases, the stuff on their plate (not talking about Nima's plate... obviously) also took time. Now, let's look at a rather extreme example, then see what we can learn from this illustration.</p>
|
||
<p><img src="https://cdn.hashnode.com/res/hashnode/image/upload/v1674533284241/8edab660-a06f-4059-8383-1145da6c39fa.png" alt class="image--center mx-auto" /></p>
|
||
<p>So this time, let's say Nima's surgery didn't go as planned and he decided to take a shower.</p>
|
||
<p><img src="https://cdn.hashnode.com/res/hashnode/image/upload/v1674533523361/7d8cc3ea-3cfe-4a0f-932f-0ac0a2a1be43.png" alt class="image--center mx-auto" /></p>
|
||
<p>I had to wait 15 minutes just to have him check his calendar and confirm his availability. Then, let's say Kan ran into a long line at the gym, so it took him forever to sign in.</p>
|
||
<p><img src="https://cdn.hashnode.com/res/hashnode/image/upload/v1674533591973/836f51ef-2bd6-4720-8cb8-e8f87c455216.png" alt class="image--center mx-auto" /></p>
|
||
<p>After waiting nearly 10 minutes, I managed to convince him to cut his routine short and just inhale a couple of times while he was heading back to the car to get his laptop. It's 30 mins already.</p>
|
||
<p>Then to top things off, I call Julie only to find out she's having intense Zoom issues and has to keep restarting Zoom.</p>
|
||
<p><img src="https://cdn.hashnode.com/res/hashnode/image/upload/v1674533661580/9ccc409e-5462-4a70-b54a-e59f1a98cd08.png" alt class="image--center mx-auto" /></p>
|
||
<p>It's another 15-minute wait before she's able to check her calendar. With almost an hour elapsed, I can guarantee the restaurant won't be taking any future calls from me.</p>
|
||
<h2 id="heading-there-must-be-a-better-way">There must be a better way</h2>
|
||
<p>As we could see in the illustration above, all the different tasks that my colleagues were doing extended the timeline for the call. In the last example, I spent an entire hour trying to schedule a simple team lunch. Because there was a lineup at the gym, Kan had to wait, which meant I had to wait, which meant the restaurant customer service rep had to wait too.</p>
|
||
<p>This is of course isn't a great situation. The poor customer service rep at the restaurant shouldn't have had to wait an hour for me to book time at the restaurant. That's an hour they could be serving other customers, generating revenue, and working towards Employee of the Month. Instead, that rep has to explain to their boss how my colleague's diaper surgery caused a significant loss of revenue to the restaurant. Or, how a potential customer's inhaling took up 20 minutes of their time.</p>
|
||
<p>This sounds pretty odd, but every time we call a remote service without proper boundaries we're doing the same thing. Often, as part of fulfilling some requests, our systems call out to remote dependencies (caches, databases, queues, internal services, and external third parties). Every dependency has the potential to induce undue strain on our system. AWS could be having an outage, or another service we depend on could be experiencing some degradation. Without proper controls in place, one problem can dramatically impact our system's availability and reliability. In this world where microservices are quite common, it's typical to have a dizzying web of dependencies involved in the fulfillment of a simple request. Uncontrolled calls can cause significant contention and impact many of the services in the web.</p>
|
||
<p>Let's look at how we improve.</p>
|
||
<h3 id="heading-issue-a-deadline-timeouts-retries-backoff-and-circuit-breakers">Issue a deadline - timeouts, retries, backoff, and circuit breakers</h3>
|
||
<h4 id="heading-timeouts">Timeouts</h4>
|
||
<p>The first concept that's critical to containing the blast radius is timeouts. If we effectively gave everyone on the call a deadline -- say a minute or two, we can control the maximum length of the call. If I have three colleagues and I'm only willing to wait 2 minutes for each person to respond, then the call shouldn't go too far beyond 6 minutes. This would reduce the wildly varying possibility that the poor employee could be waiting anywhere between a few minutes to hours for me to complete my booking. This also contains the impact of any one person's tasks on our scheduling. If we have a cutoff then Nima's surgeries won't bubble up, causing me to wait, which pushes the restaurant employee further from that Employee of the Month Award.</p>
|
||
<p>In the age of microservices, transient failures and prolonged outages are quite common. One system may experience an influx of requests they weren't prepared to manage. Or, someone could have rolled out a faulty change at a third party that caused our requests to take longer, or fail. Any and all of these changes can cascade if they're not controlled. When we use timeouts, we dampen the impact of these stresses and strains. If the cache is experiencing some degradation, applying a timeout means we wait up to a specified maximum then we abandon the call. This ensures that a dependency failure (e.g. a cache failing) doesn't impact our consumers, or our consumers' consumers, and so on. By stopping the propagation we ensure this fault doesn't spread across the web of microservices and bring others down.</p>
|
||
<h4 id="heading-retries-and-backoff">Retries and Backoff</h4>
|
||
<p>Another approach that's closely related to timeouts is Retries. If I called and realized that my colleague is busy, I could wait until the deadline was exceeded and then offer to call back in a few minutes. Chances are they might be less tied up in a few minutes. We can take a similar approach with our systems. If we reach out to a dependency and don't get a reply before the timeout, we can retry after a short delay. Typically that retry could be a few milliseconds. We can also set a maximum number of retries. Many dependency issues are transient. In such cases, retrying after a short delay increases the likelihood of a successful response. Placing a cap on the number of retries guarantees that prolonged issues don't cascade. If there's an outage, retries might likely fail too. Setting the right number of retries gives us the right balance between making our consumers wait, or failing early with an error.</p>
|
||
<p>When we timeout and then delay for a fixed interval, it could compound the problem if our dependency is under heavy load. Imagine some dependency we're trying to reach is buckling under heavy load. If every caller is timing out and then retrying at a similar interval, the system could end up getting surges called a Thundering Herd. Effectively, everyone tries to get in, waits for a short delay, then everyone tries to get in again (roughly) at the same time. That complicates recovery for the target system. The system doesn't get to recover properly due to surges hitting it by clients retrying at similar intervals. Instead of using a fixed interval, we can randomize the delay before retrying. This helps breaks the synchronicity with other clients and effectively reduces the net load on the target server.</p>
|
||
<p>Taking this strategy one step further, we can increase the time we're willing to wait with each retry. In particular, we can increase the maximum random delay exponentially. This allows us to provide more room for the target system to recover. If the problem is indeed transient, allowing the dependency some breathing room might be just what it needs to recover from whatever degradation is occurring. This is widely known as Random Exponential Backoff. We're backing off after each request and calculating a random delay that we're willing to wait. On each retry, we increase the maximum possible time exponentially. This can really help in reducing the load on a target system. Often load from consumers compounds issues a system faces, so using Random Exponential Backoff helps to alleviate some of the load, allowing a system to recover.</p>
|
||
<h4 id="heading-circuit-breakers">Circuit Breakers</h4>
|
||
<p>Finally, if I noticed I'd been unsuccessful in getting to one of my colleagues, I could make a decision to stop calling. Instead of continuously retrying, I could effectively postpone all attempts to call until after I've surpassed a fixed number of retries. This is another common strategy that helps to prevent failures from cascading. By making a firm decision to stop forwarding requests to a suboptimal dependency, we're preventing any degradation from impacting any of our consumers. Instead of continuing to wait, which means our consumers have to wait, we can just fail gracefully, and earlier with an appropriate error. Unfortunately, this does often mean loss of functionality for our customers, but the upside is the maintenance of availability and reliability. By choosing not to forward requests to a borked dependency, we are removing lots of potential contention on the network that could impact many other areas of our service. We're ensuring one bad apple doesn't spoil the entire bunch.</p>
|
||
<h2 id="heading-conclusion">Conclusion</h2>
|
||
<p>And that's a wrap! I hope you enjoyed this post about cascading failures and how we can mitigate them in our systems. If we're not careful failures can cascade and have a sustained impact on our systems and impact our customers. The discussed strategies help us combat these issues while providing a graceful experience for our customers! Thanks for stopping by!</p>
|
||
<p>I'd love to hear some of your stories about how these strategies worked for you! Let's chat!</p>
|
||
</div></div></div></div><div class="max-w-2xl mx-auto border-t border-border pt-8 space-y-5"><div class="flex flex-wrap items-center gap-2"><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/reliability">#<!-- -->reliability</a><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/timeout">#<!-- -->timeout</a><a class="inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary" href="/tag/high-availability">#<!-- -->high-availability</a></div></div></div></article></div></div></div></section><section class="pb-8"><div class="px-4 sm:px-8"><div class="w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="max-w-4xl w-full mx-auto"><div class="rounded-xl border bg-card text-card-foreground p-5 sm:p-8"><section class="space-y-5"><div class="flex items-center justify-between"><h2 class="text-lg font-bold tracking-tight">Comments</h2><a target="_blank" rel="noopener" class="inline-flex items-center gap-2 rounded-md border border-border px-3 py-1.5 text-sm font-medium text-foreground transition hover:border-primary hover:text-primary" href="https://hashnode.com/posts/cascading-failures-and-the-impossibility-of-scheduling-team-lunches/63e4e3f3b9f58f9b3c72ce21"><i class="fa-regular fa-comment" aria-hidden="true"></i>Join the discussion</a></div><p class="text-sm text-muted-foreground py-5 text-center">No comments yet.<!-- --> <a target="_blank" rel="noopener" class="text-primary hover:underline" href="https://hashnode.com/posts/cascading-failures-and-the-impossibility-of-scheduling-team-lunches/63e4e3f3b9f58f9b3c72ce21">Be the first to comment.</a></p></section></div></div></div></div></section><section class="pb-8"><div class="px-4 sm:px-8"><div class="w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]"><div class="max-w-4xl w-full mx-auto"><section class="space-y-6"><h2 class="text-2xl font-bold tracking-tight text-balance">More from this blog</h2><div class="flex flex-col gap-6"><article aria-labelledby="_R_panpfalb_" class="h-full"><a class="block h-full" href="/instrumenting-at-10-years-per-second"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_panpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Instrumenting at 10 years per second</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">How we made custom instrumentation blazing fast, simple, and data-centric</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Dec 23, 2025</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>8 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Instrumenting at 10 years per second" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iRnUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg"/></div></div></div></div></a></article><article aria-labelledby="_R_19anpfalb_" class="h-full"><a class="block h-full" href="/on-shipping-smaller-prs"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_19anpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">On shipping smaller PRs</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">Concrete examples for how to ship smaller PRs</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Feb 17, 2025</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>8 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="On shipping smaller PRs" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/o9oQaOGpLz0/upload/9e19fbfd378d65c448f7233b2a273b79.jpeg"/></div></div></div></div></a></article><article aria-labelledby="_R_1panpfalb_" class="h-full"><a class="block h-full" href="/small-prs-ship-faster-qed"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_1panpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Small PRs ship faster - QED</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">My unrestrained passion for shipping smaller PRs constrained into a handful of benefits</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Dec 25, 2024</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>5 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Small PRs ship faster - QED" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/JzMS3RLSZn8/upload/a18ee8f82d0d56644ab82f6c1bead8b4.jpeg"/></div></div></div></div></a></article><article aria-labelledby="_R_29anpfalb_" class="h-full"><a class="block h-full" href="/order-matters-making-a-compound-index-50x-faster"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_29anpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Order matters - making a compound index 50x faster</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">A story of how we made a compound index 50x faster by re-organizing the fields</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Oct 17, 2024</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>6 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Order matters - making a compound index 50x faster" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/upload/v1726107899370/04b1ac6d-7825-4a05-b5d0-0be78a2f49d9.png"/></div></div></div></div></a></article><article aria-labelledby="_R_2panpfalb_" class="h-full"><a class="block h-full" href="/dear-editor-we-need-better-database-observability"><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 h-full"><div class="grid h-full gap-4 p-4 items-stretch md:gap-6 md:p-6 md:grid-cols-[1.2fr_1fr]"><div class="flex flex-col gap-4"><header class="flex flex-col gap-3"><h3 id="_R_2panpfalb_" class="text-foreground text-lg sm:text-xl font-bold leading-snug tracking-tight transition-colors line-clamp-3">Dear Editor: We need better Database Observability</h3><p class="text-sm leading-relaxed text-muted-foreground line-clamp-3">A call for better database observability, or awareness to a better reality</p></header><div class="flex flex-wrap items-center gap-3 text-sm text-muted-foreground"><span class="whitespace-nowrap">Sep 12, 2024</span><span class="whitespace-nowrap"><span class="mr-3" aria-hidden="true">·</span>8 min read</span></div></div><div class="flex items-center md:justify-end order-first md:order-last"><div class="relative overflow-hidden rounded-lg bg-muted aspect-[800/420] w-full md:max-w-[320px]"><img alt="Dear Editor: We need better Database Observability" loading="lazy" decoding="async" data-nimg="fill" class="object-cover object-center" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" src="https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/NvFkYV2ngOk/upload/2f4cbc39f32355a334d9119af2dd086c.jpeg"/></div></div></div></div></a></article><div class="rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 p-5 flex flex-col"><div class="flex flex-col items-center text-center gap-5"><div class="flex flex-col items-center gap-3"><div class="flex h-16 w-16 shrink-0 items-center justify-center rounded-full bg-muted"><i class="text-2xl fa-brands fa-hashnode text-foreground" aria-hidden="true"></i></div><div class="space-y-1"><p class="text-base font-semibold text-foreground leading-tight">Perspective Unspoken</p><p class="text-sm text-muted-foreground">19<!-- --> posts</p></div></div><div class="prose prose-sm dark:prose-invert max-w-xs mx-auto text-muted-foreground"><p>Welcome to "Perspective Unspoken", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.</p>
|
||
</div><div class="flex flex-wrap items-center justify-center gap-3"><a href="https://twitter.com/kramnaej" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="Twitter"><i class="fa-brands fa-x-twitter" aria-hidden="true"></i></a><a href="https://github.com/jaywhy13" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="GitHub"><i class="fa-brands fa-github" aria-hidden="true"></i></a><a href="https://www.linkedin.com/in/jean-mark-wright/" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="LinkedIn"><i class="fa-brands fa-linkedin" aria-hidden="true"></i></a><a href="https://hashnode.com/@jaywhy13" target="_blank" rel="noopener noreferrer nofollow ugc" class="text-muted-foreground transition-colors hover:text-foreground" aria-label="Hashnode"><i class="fa-brands fa-hashnode" aria-hidden="true"></i></a></div></div></div></div></section></div></div></div></section><script type="application/ld+json">{"@context":"https://schema.org","@type":"Article","url":"https://jaywhy13.hashnode.dev/cascading-failures-and-the-impossibility-of-scheduling-team-lunches","mainEntityOfPage":"https://jaywhy13.hashnode.dev/cascading-failures-and-the-impossibility-of-scheduling-team-lunches","headline":"Cascading failures and the impossibility of scheduling team lunches","description":"Introduction\nIn this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't...","author":{"@type":"Person","name":"Jean-Mark Wright","url":"https://hashnode.com/@jaywhy13","image":"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg","sameAs":"https://x.com/kramnaej"},"publisher":{"@type":"Organization","name":"Perspective Unspoken","url":"https://jaywhy13.hashnode.dev"},"datePublished":"2023-02-09T12:15:47.579Z","dateModified":"2023-02-09T12:15:47.579Z","image":{"@type":"ImageObject","url":"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg"},"keywords":"Reliability, timeout, high availability"}</script><footer class="px-4 pt-8 pb-20 sm:px-8 text-muted-foreground" aria-label="Footer"><div class="w-full max-w-4xl mx-auto"><div class="flex flex-col items-center gap-4 text-center text-sm"><p class="font-medium"><time dateTime="2026"><span class="font-sans">©</span> <!-- -->2026</time> <!-- -->Perspective Unspoken</p><nav aria-label="Footer navigation"><ul class="flex flex-wrap justify-center gap-4 text-sm tracking-wide"><li><a class="transition hover:text-foreground" href="/archive">Archive</a></li><li><a class="transition hover:text-foreground" target="_blank" rel="noopener" href="https://hashnode.com/privacy-policy">Privacy</a></li><li><a class="transition hover:text-foreground" target="_blank" rel="noopener" href="https://hashnode.com/terms">Terms</a></li></ul></nav><div class="pt-8 flex justify-center gap-6"><a target="_blank" rel="noopener" class="flex items-center gap-2 text-sm transition hover:text-foreground" title="Sitemap" href="/sitemap.xml"><i class="fa-regular fa-sitemap"></i><span>Sitemap</span></a><a target="_blank" rel="noopener" class="flex items-center gap-2 text-sm transition hover:text-foreground" title="RSS Feed" href="/rss.xml"><i class="fa-regular fa-rss"></i><span>RSS</span></a></div><div class="pt-10 text-center"><a target="_blank" rel="noopener" aria-label="Visit Hashnode (opens in a new tab)" class="inline-flex items-center justify-center gap-2 whitespace-nowrap font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 hover:bg-accent hover:text-accent-foreground h-10 rounded-md text-xl px-5 py-4" href="https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev&utm_medium=referral&utm_campaign=blog_footer_logo&utm_content=logo"><svg fill="none" viewBox="0 0 282 39" aria-hidden="true" class="h-6 w-auto min-w-[9.5rem] text-muted-foreground transition hover:text-foreground"><path fill="currentColor" d="M2.68 13.032c-3.573 3.505-3.573 9.363 0 12.936L13.032 36.32c3.505 3.573 9.363 3.573 12.936 0L36.32 25.968c3.573-3.573 3.573-9.431 0-12.936L25.968 2.68c-3.573-3.573-9.431-3.573-12.936 0zm12.211 1.935c2.507-2.521 6.582-2.544 9.104-.038s2.544 6.582.038 9.104-6.582 2.544-9.104.038-2.544-6.582-.038-9.104"></path><path fill="currentColor" d="M268.379 39q-6.322 0-10.35-4.108-4.002-4.134-4.002-10.483 0-6.296 4.002-10.43 4-4.135 10.163-4.135 6.028 0 9.817 3.734 3.814 3.735 3.814 9.497v2.8h-24.248v-4.054h17.633q-.187-2.747-2.054-4.508-1.868-1.76-4.962-1.76-3.28-.001-5.388 2.374-2.108 2.373-2.108 6.375 0 4.188 2.348 6.695 2.347 2.481 5.895 2.481 1.68 0 2.934-.453a8.5 8.5 0 0 0 2.321-1.307q1.041-.854 2.268-2.614l4.694 2.96q-1.707 2.748-3.494 4.135a12.7 12.7 0 0 1-3.975 2.081q-2.187.72-5.308.72m-33.496 0q-5.575 0-9.283-4.028-3.681-4.029-3.681-10.484 0-6.43 3.761-10.51 3.76-4.108 9.47-4.108 2.8 0 5.175 1.2 2.4 1.174 3.628 3.148l.026-.08V0h7.016v38.2h-6.722v-4.722l.64.8-.694.133q-1.28 2.108-3.788 3.361Q237.924 39 234.883 39m1.627-6.002q3.308 0 5.495-2.428t2.188-6.135q0-3.762-2.214-6.109-2.214-2.374-5.469-2.374-3.28 0-5.522 2.347-2.214 2.348-2.214 6.163 0 3.76 2.214 6.162 2.214 2.374 5.522 2.374M205.811 39q-6.376 0-10.537-4.161-4.161-4.188-4.161-10.404 0-6.321 4.161-10.456 4.188-4.135 10.537-4.135 6.348 0 10.537 4.161 4.188 4.161 4.188 10.43 0 6.216-4.188 10.404Q212.186 39 205.811 39m0-6.162q3.307 0 5.575-2.374 2.267-2.374 2.267-6.055 0-3.707-2.294-6.056-2.267-2.347-5.548-2.347-3.308 0-5.575 2.374-2.268 2.348-2.268 6.029t2.241 6.055q2.268 2.374 5.602 2.374M162.586 38.2V10.644h6.802v5.015l-.8-.987h.906q1.308-2.187 3.762-3.494 2.48-1.335 5.228-1.334 4.615 0 7.256 2.8 2.667 2.776 2.667 7.603V38.2h-7.069V21.794q0-2.854-1.387-4.401t-4.134-1.547q-2.614 0-4.402 1.974-1.76 1.974-1.76 4.881V38.2zm-30.181 0V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.069V21.794q0-2.827-1.387-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.814 2.027-1.814 4.962v15.391zm-14.416.8q-2.882 0-5.015-.64-2.108-.614-3.761-1.84-1.654-1.228-2.668-2.695l-.987-1.467 4.695-3.174.773 1.067a9.1 9.1 0 0 0 1.788 1.814 8.2 8.2 0 0 0 2.32 1.2q1.308.4 3.068.4 2.161 0 3.441-.8 1.308-.8 1.307-2.294 0-1.36-1.147-2.108-1.12-.747-5.068-1.654-5.335-1.253-7.656-3.334-2.32-2.107-2.321-5.362 0-3.76 3.148-6.002 3.148-2.267 7.923-2.267 2.454 0 4.348.507t3.388 1.493q1.493.988 2.454 2.188l.987 1.2-4.482 3.255-.667-.774a7.4 7.4 0 0 0-1.627-1.387 6.4 6.4 0 0 0-2.054-.88 9.5 9.5 0 0 0-2.401-.294q-1.974 0-3.201.8-1.2.774-1.2 1.921 0 1.254 1.12 2.027 1.147.748 5.229 1.681 5.334 1.227 7.629 3.308 2.294 2.055 2.294 5.308 0 4.029-3.228 6.43Q123.217 39 117.989 39M87.61 39q-4.508 0-7.335-2.454-2.828-2.481-2.828-6.482 0-4.188 3.254-6.669t8.483-2.48q2.082 0 4.028.426 1.974.427 3.361 1.12v-1.6q0-2.588-1.813-4.082-1.788-1.493-4.615-1.493-1.628 0-2.801.373a8.7 8.7 0 0 0-2.188.987q-.987.614-1.84 1.414l-.854.8-4.081-3.788 1.307-1.173a18.7 18.7 0 0 1 2.934-2.108q1.601-.933 3.601-1.44t4.589-.507q5.868 0 9.229 2.88 3.361 2.882 3.361 8.084V38.2h-6.749v-5.922l1.654 2.187h-1.76q-1.308 2.055-3.655 3.308Q90.545 39 87.611 39m1.814-4.988q3.148 0 5.149-2.187 2-2.215 2-5.309v-.08q-1.12-.693-2.72-1.094a13 13 0 0 0-3.388-.426q-2.828 0-4.562 1.2-1.734 1.173-1.734 3.521 0 2.028 1.468 3.201t3.787 1.174M49 38.2V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.07V21.794q0-2.827-1.386-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.815 2.027-1.814 4.962v15.391z"></path></svg></a></div></div></div></footer></main><!--$--><!--/$--><script src="/_next/static/chunks/webpack-d716a75e4017380e.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[16174,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"ThemeProvider\"]\n3:I[82180,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"TooltipProvider\"]\n4:I[63851,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"UrqlProvider\"]\n5:I[82714,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"ConsentManager\"]\n6:I[9766,[],\"\"]\n7:I[98924,[],\"\"]\n8:I[33293,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"4345\",\"static/chunks/app/not-found-b861935655c5698b.js\"],\"default\"]\n9:I[21447,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"Integrations\"]\na:I[35514,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3874\",\"static/chunks/3874-323555cd5c106ba4.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"7177\",\"static/chunks/app/layout-f61a55efa656d91f.js\"],\"Analytics\"]\nc:I[24431,[],\"OutletBoundary\"]\ne:I[15278,[],\"AsyncMetadataOutlet\"]\n10:I[24431,[],\"ViewportBoundary\"]\n12:I[24431,[],\"MetadataBoundary\"]\n13:\"$Sreact.suspense\"\n15:I[57150,[],\"\"]\n:HL[\"/_next/static/media/304eba2dc288a929-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/38077b822d9512d7-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/40ddcd2db5e33048-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/707228f0d4113e57-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/a73fce2732cf3a48-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/c25783985a269397-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/c86b497a7be35730-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/d9cd7618114896c5-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/f4e6f3e26e87864d-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/fde6a32e2c53c69f-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/css/bc5c0eaecd39bc7d.css\",\"style\"]\n:HL[\"/_next/static/css/7ffcc02a3c67389b.css\",\"style\"]\n:HL[\"/_next/static/css/c20e76e7d11dec32.css\",\"style\"]\n:HL[\"/_next/static/css/a6eb8c1f37be68d5.css\",\"style\"]\n0:{\"P\":null,\"b\":\"ePiaRevyKQ9kV7jt41IDP\",\"p\":\"\",\"c\":[\"\",\"cascading-failures-and-the-impossibility-of-scheduling-team-lunches\"],\"i\":false,\"f\":[[[\"\",{\"children\":[[\"slug\",\"cascading-failures-and-the-impossibility-of-scheduling-team-lunches\",\"c\"],{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],[\"\",[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/bc5c0eaecd39bc7d.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/7ffcc02a3c67389b.css\",\"precedence\":"])</script><script>self.__next_f.push([1,"\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"2\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/c20e76e7d11dec32.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"3\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/a6eb8c1f37be68d5.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"suppressHydrationWarning\":true,\"children\":[[\"$\",\"head\",null,{\"children\":[\"$\",\"script\",null,{\"type\":\"text/javascript\",\"dangerouslySetInnerHTML\":{\"__html\":\"window.addEventListener('message',function(e){var d=e.data;if(!d||d.__hnWidget!==1||typeof d.h!=='number'||d.h\u003c=0)return;var f=document.querySelectorAll('.hn-widget-frame');for(var i=0;i\u003cf.length;i++){if(f[i].contentWindow===e.source){f[i].style.height=d.h+'px';break;}}});\"}}]}],[\"$\",\"body\",null,{\"className\":\"__variable_ede2c9 __variable_558dcf bg-background dark:bg-background\",\"children\":[\"$\",\"$L2\",null,{\"attribute\":\"class\",\"defaultTheme\":\"system\",\"enableSystem\":true,\"disableTransitionOnChange\":true,\"children\":[\"$\",\"$L3\",null,{\"children\":[\"$\",\"$L4\",null,{\"children\":[\"$\",\"$L5\",null,{\"children\":[[\"$\",\"$L6\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L7\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"$L8\",null,{}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}],[\"$\",\"$L9\",null,{}],[\"$\",\"$La\",null,{}]]}]}]}]}]}]]}]]}],{\"children\":[[\"slug\",\"cascading-failures-and-the-impossibility-of-scheduling-team-lunches\",\"c\"],[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L6\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L7\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[\"__PAGE__\",[\"$\",\"$1\",\"c\",{\"children\":[\"$Lb\",null,[\"$\",\"$Lc\",null,{\"children\":[\"$Ld\",[\"$\",\"$Le\",null,{\"promise\":\"$@f\"}]]}]]}],{},null,false]},null,false]},null,false],[\"$\",\"$1\",\"h\",{\"children\":[null,[[\"$\",\"$L10\",null,{\"children\":\"$L11\"}],[\"$\",\"meta\",null,{\"name\":\"next-size-adjust\",\"content\":\"\"}]],[\"$\",\"$L12\",null,{\"children\":[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$13\",null,{\"fallback\":null,\"children\":\"$L14\"}]}]}]]}],false]],\"m\":\"$undefined\",\"G\":[\"$15\",[]],\"s\":false,\"S\":false}\n11:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}]]\nd:null\n16:I[80622,[],\"IconMark\"]\nf:{\"metadata\":[[\"$\",\"title\",\"0\",{\"children\":\"Cascading failures and the impossibility of scheduling team lunches\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"Introduction\\nIn this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't...\"}],[\"$\",\"link\",\"2\",{\"rel\":\"canonical\",\"href\":\"https://jaywhy13.hashnode.dev/cascading-failures-and-the-impossibility-of-scheduling-team-lunches\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:title\",\"content\":\"Cascading failures and the impossibility of scheduling team lunches\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:description\",\"content\":\"Introduction\\nIn this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't...\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:url\",\"content\":\"https://jaywhy13.hashnode.dev/cascading-failures-and-the-impossibility-of-scheduling-team-lunches\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:site_name\",\"content\":\"Perspective Unspoken\"}],[\"$\",\"meta\",\"7\",{\"property\":\"og:image\",\"content\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg\"}],[\"$\",\"meta\",\"8\",{\"property\":\"og:image:width\",\"content\":\"1200\"}],[\"$\",\"meta\",\"9\",{\"property\":\""])</script><script>self.__next_f.push([1,"og:image:height\",\"content\":\"630\"}],[\"$\",\"meta\",\"10\",{\"property\":\"og:type\",\"content\":\"article\"}],[\"$\",\"meta\",\"11\",{\"property\":\"article:published_time\",\"content\":\"2023-02-09T12:15:47.579Z\"}],[\"$\",\"meta\",\"12\",{\"property\":\"article:author\",\"content\":\"https://jaywhy13.hashnode.dev/@jaywhy13\"}],[\"$\",\"meta\",\"13\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"14\",{\"name\":\"twitter:creator\",\"content\":\"https://twitter.com/kramnaej\"}],[\"$\",\"meta\",\"15\",{\"name\":\"twitter:title\",\"content\":\"Cascading failures and the impossibility of scheduling team lunches\"}],[\"$\",\"meta\",\"16\",{\"name\":\"twitter:description\",\"content\":\"Introduction\\nIn this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't...\"}],[\"$\",\"meta\",\"17\",{\"name\":\"twitter:image\",\"content\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg\"}],[\"$\",\"link\",\"18\",{\"rel\":\"icon\",\"href\":\"/favicon.ico\"}],[\"$\",\"$L16\",\"19\",{}]],\"error\":null,\"digest\":\"$undefined\"}\n14:\"$f:metadata\"\n17:I[652,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"8974\",\"static/chunks/app/page-cc18ae1a3a7a5ed8.js\"],\"BlogHeader\"]\n18:T31cb,\u003ch2 id=\"heading-introduction\"\u003eIntroduction\u003c/h2\u003e\n\u003cp\u003eIn this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't want that failure propagating, or cascading to other areas. You must control the blast radius, or everything will go up in flames. Ideally, you want to ensure that your components absorb shocks and strains and don't provide a pathway for propagation. One uncontained fault can jeopardize your system's availability and reliability. We can use timeouts, circuit breakers, and other techniques to reduce the impact.\u003c/p\u003e\n\u003ch2 id=\"heading-its-lunchtime\"\u003eIt's lunchtime!\u003c/h2\u003e\n\u003cp\u003eTo illustrate the problem, we'll consider the task of scheduling team lunch for a rather busy set of colleagues. Let's assume everybody is working remotely and tends to have a life, which gets in the way of team initiatives. This makes scheduling a rather interesting adventure. Nobody pays attention to the polls in Slack to decide on a time for lunch, so we have to resort to calling everyone manually. For our consideration, let's assume I called the restaurant and was fortunate enough to get a patient representative who waited while I called my other team members to book.\u003c/p\u003e\n\u003cp\u003eHave a look at the diagram below. Hopefully, it looks familiar. It's a trace graph! Each box represents some action that's happening. The first box represents the duration of my call to the restaurant. The boxes in the next line represent each co-worker I'm calling in an attempt to schedule the team lunch. Each box below represents the life commitments that make scheduling a time-consuming affair. Let's take a closer look!\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1674532131705/8577af52-25a5-4e67-aaec-029cd19d8ee7.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eSo I called the restaurant and asked them to hold while I connected with my colleagues and confirmed. First up, I got a Pakistani colleague we'll call Nima. He has a beautiful baby girl. When I called he just happened to be in the middle of a diaper change. Yikes. After surgically removing the diaper and installing a clean one, it took him a few more minutes to get back to his computer and check his calendar. He's free! Next up, Kan, a health-conscious teammate who faithfully and consistently"])</script><script>self.__next_f.push([1," goes to the gym once or twice every couple of months. Just my luck! Gym time just happens to be now. I had to wait until he signed in and did his intensive workout session consisting of inhaling and stretching. 15 minutes in, he's able to get back to his computer and check his schedule, he's free! Finally, let's check in with Julie. It's almost 20 minutes and I can hear the receptionist on the line getting impatient. She's having issues getting Zoom started, and she's about to head into a quick meeting. Close to the half-hour mark, Julie confirms that she too is free! Overjoyed, the receptionist books us in and terminates the call.\u003c/p\u003e\n\u003cp\u003ePhew... that only took 30 minutes. The receptionist had to wait for me to connect with all my colleagues. All of them had different things on their plate and were unable to respond immediately. In many cases, the stuff on their plate (not talking about Nima's plate... obviously) also took time. Now, let's look at a rather extreme example, then see what we can learn from this illustration.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1674533284241/8edab660-a06f-4059-8383-1145da6c39fa.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eSo this time, let's say Nima's surgery didn't go as planned and he decided to take a shower.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1674533523361/7d8cc3ea-3cfe-4a0f-932f-0ac0a2a1be43.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eI had to wait 15 minutes just to have him check his calendar and confirm his availability. Then, let's say Kan ran into a long line at the gym, so it took him forever to sign in.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1674533591973/836f51ef-2bd6-4720-8cb8-e8f87c455216.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eAfter waiting nearly 10 minutes, I managed to convince him to cut his routine short and just inhale a couple of times while he was heading back to the car to get his laptop. It's 30 mins already.\u003c/p\u003e\n\u003cp\u003eThen to top things off, I call Julie only to find out she's having intense Zoom issues and has to keep restarting Zoom.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1674533661580/9ccc409e-5462-4a70-b54a-e59f1a98cd08.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eIt's another 15-minute wait before she's able to check her calendar. With almost an hour elapsed, I can guarantee the restaurant won't be taking any future calls from me.\u003c/p\u003e\n\u003ch2 id=\"heading-there-must-be-a-better-way\"\u003eThere must be a better way\u003c/h2\u003e\n\u003cp\u003eAs we could see in the illustration above, all the different tasks that my colleagues were doing extended the timeline for the call. In the last example, I spent an entire hour trying to schedule a simple team lunch. Because there was a lineup at the gym, Kan had to wait, which meant I had to wait, which meant the restaurant customer service rep had to wait too.\u003c/p\u003e\n\u003cp\u003eThis is of course isn't a great situation. The poor customer service rep at the restaurant shouldn't have had to wait an hour for me to book time at the restaurant. That's an hour they could be serving other customers, generating revenue, and working towards Employee of the Month. Instead, that rep has to explain to their boss how my colleague's diaper surgery caused a significant loss of revenue to the restaurant. Or, how a potential customer's inhaling took up 20 minutes of their time.\u003c/p\u003e\n\u003cp\u003eThis sounds pretty odd, but every time we call a remote service without proper boundaries we're doing the same thing. Often, as part of fulfilling some requests, our systems call out to remote dependencies (caches, databases, queues, internal services, and external third parties). Every dependency has the potential to induce undue strain on our system. AWS could be having an outage, or another service we depend on could be experiencing some degradation. Without proper controls in place, one problem can dramatically impact our system's availability and reliability. In this world where microservices are quite common, it's typical to have a dizzying web "])</script><script>self.__next_f.push([1,"of dependencies involved in the fulfillment of a simple request. Uncontrolled calls can cause significant contention and impact many of the services in the web.\u003c/p\u003e\n\u003cp\u003eLet's look at how we improve.\u003c/p\u003e\n\u003ch3 id=\"heading-issue-a-deadline-timeouts-retries-backoff-and-circuit-breakers\"\u003eIssue a deadline - timeouts, retries, backoff, and circuit breakers\u003c/h3\u003e\n\u003ch4 id=\"heading-timeouts\"\u003eTimeouts\u003c/h4\u003e\n\u003cp\u003eThe first concept that's critical to containing the blast radius is timeouts. If we effectively gave everyone on the call a deadline -- say a minute or two, we can control the maximum length of the call. If I have three colleagues and I'm only willing to wait 2 minutes for each person to respond, then the call shouldn't go too far beyond 6 minutes. This would reduce the wildly varying possibility that the poor employee could be waiting anywhere between a few minutes to hours for me to complete my booking. This also contains the impact of any one person's tasks on our scheduling. If we have a cutoff then Nima's surgeries won't bubble up, causing me to wait, which pushes the restaurant employee further from that Employee of the Month Award.\u003c/p\u003e\n\u003cp\u003eIn the age of microservices, transient failures and prolonged outages are quite common. One system may experience an influx of requests they weren't prepared to manage. Or, someone could have rolled out a faulty change at a third party that caused our requests to take longer, or fail. Any and all of these changes can cascade if they're not controlled. When we use timeouts, we dampen the impact of these stresses and strains. If the cache is experiencing some degradation, applying a timeout means we wait up to a specified maximum then we abandon the call. This ensures that a dependency failure (e.g. a cache failing) doesn't impact our consumers, or our consumers' consumers, and so on. By stopping the propagation we ensure this fault doesn't spread across the web of microservices and bring others down.\u003c/p\u003e\n\u003ch4 id=\"heading-retries-and-backoff\"\u003eRetries and Backoff\u003c/h4\u003e\n\u003cp\u003eAnother approach that's closely related to timeouts is Retries. If I called and realized that my colleague is busy, I could wait until the deadline was exceeded and then offer to call back in a few minutes. Chances are they might be less tied up in a few minutes. We can take a similar approach with our systems. If we reach out to a dependency and don't get a reply before the timeout, we can retry after a short delay. Typically that retry could be a few milliseconds. We can also set a maximum number of retries. Many dependency issues are transient. In such cases, retrying after a short delay increases the likelihood of a successful response. Placing a cap on the number of retries guarantees that prolonged issues don't cascade. If there's an outage, retries might likely fail too. Setting the right number of retries gives us the right balance between making our consumers wait, or failing early with an error.\u003c/p\u003e\n\u003cp\u003eWhen we timeout and then delay for a fixed interval, it could compound the problem if our dependency is under heavy load. Imagine some dependency we're trying to reach is buckling under heavy load. If every caller is timing out and then retrying at a similar interval, the system could end up getting surges called a Thundering Herd. Effectively, everyone tries to get in, waits for a short delay, then everyone tries to get in again (roughly) at the same time. That complicates recovery for the target system. The system doesn't get to recover properly due to surges hitting it by clients retrying at similar intervals. Instead of using a fixed interval, we can randomize the delay before retrying. This helps breaks the synchronicity with other clients and effectively reduces the net load on the target server.\u003c/p\u003e\n\u003cp\u003eTaking this strategy one step further, we can increase the time we're willing to wait with each retry. In particular, we can increase the maximum random delay exponentially. This allows us to provide more room for the target system to recover. If the problem is indeed transient, allowing the dependency some breath"])</script><script>self.__next_f.push([1,"ing room might be just what it needs to recover from whatever degradation is occurring. This is widely known as Random Exponential Backoff. We're backing off after each request and calculating a random delay that we're willing to wait. On each retry, we increase the maximum possible time exponentially. This can really help in reducing the load on a target system. Often load from consumers compounds issues a system faces, so using Random Exponential Backoff helps to alleviate some of the load, allowing a system to recover.\u003c/p\u003e\n\u003ch4 id=\"heading-circuit-breakers\"\u003eCircuit Breakers\u003c/h4\u003e\n\u003cp\u003eFinally, if I noticed I'd been unsuccessful in getting to one of my colleagues, I could make a decision to stop calling. Instead of continuously retrying, I could effectively postpone all attempts to call until after I've surpassed a fixed number of retries. This is another common strategy that helps to prevent failures from cascading. By making a firm decision to stop forwarding requests to a suboptimal dependency, we're preventing any degradation from impacting any of our consumers. Instead of continuing to wait, which means our consumers have to wait, we can just fail gracefully, and earlier with an appropriate error. Unfortunately, this does often mean loss of functionality for our customers, but the upside is the maintenance of availability and reliability. By choosing not to forward requests to a borked dependency, we are removing lots of potential contention on the network that could impact many other areas of our service. We're ensuring one bad apple doesn't spoil the entire bunch.\u003c/p\u003e\n\u003ch2 id=\"heading-conclusion\"\u003eConclusion\u003c/h2\u003e\n\u003cp\u003eAnd that's a wrap! I hope you enjoyed this post about cascading failures and how we can mitigate them in our systems. If we're not careful failures can cascade and have a sustained impact on our systems and impact our customers. The discussed strategies help us combat these issues while providing a graceful experience for our customers! Thanks for stopping by!\u003c/p\u003e\n\u003cp\u003eI'd love to hear some of your stories about how these strategies worked for you! Let's chat!\u003c/p\u003e\n19:T2f64,## Introduction\n\nIn this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't want that failure propagating, or cascading to other areas. You must control the blast radius, or everything will go up in flames. Ideally, you want to ensure that your components absorb shocks and strains and don't provide a pathway for propagation. One uncontained fault can jeopardize your system's availability and reliability. We can use timeouts, circuit breakers, and other techniques to reduce the impact.\n\n## It's lunchtime!\n\nTo illustrate the problem, we'll consider the task of scheduling team lunch for a rather busy set of colleagues. Let's assume everybody is working remotely and tends to have a life, which gets in the way of team initiatives. This makes scheduling a rather interesting adventure. Nobody pays attention to the polls in Slack to decide on a time for lunch, so we have to resort to calling everyone manually. For our consideration, let's assume I called the restaurant and was fortunate enough to get a patient representative who waited while I called my other team members to book.\n\nHave a look at the diagram below. Hopefully, it looks familiar. It's a trace graph! Each box represents some action that's happening. The first box represents the duration of my call to the restaurant. The boxes in the next line represent each co-worker I'm calling in an attempt to schedule the team lunch. Each box below represents the life commitments that make scheduling a time-consuming affair. Let's take a closer look!\n\n\n\nSo I called the restaurant and asked them to hold while I connected with my colleagues and confirmed. First up, I got a Pakistani colleague we'll c"])</script><script>self.__next_f.push([1,"all Nima. He has a beautiful baby girl. When I called he just happened to be in the middle of a diaper change. Yikes. After surgically removing the diaper and installing a clean one, it took him a few more minutes to get back to his computer and check his calendar. He's free! Next up, Kan, a health-conscious teammate who faithfully and consistently goes to the gym once or twice every couple of months. Just my luck! Gym time just happens to be now. I had to wait until he signed in and did his intensive workout session consisting of inhaling and stretching. 15 minutes in, he's able to get back to his computer and check his schedule, he's free! Finally, let's check in with Julie. It's almost 20 minutes and I can hear the receptionist on the line getting impatient. She's having issues getting Zoom started, and she's about to head into a quick meeting. Close to the half-hour mark, Julie confirms that she too is free! Overjoyed, the receptionist books us in and terminates the call.\n\nPhew... that only took 30 minutes. The receptionist had to wait for me to connect with all my colleagues. All of them had different things on their plate and were unable to respond immediately. In many cases, the stuff on their plate (not talking about Nima's plate... obviously) also took time. Now, let's look at a rather extreme example, then see what we can learn from this illustration.\n\n\n\nSo this time, let's say Nima's surgery didn't go as planned and he decided to take a shower.\n\n\n\nI had to wait 15 minutes just to have him check his calendar and confirm his availability. Then, let's say Kan ran into a long line at the gym, so it took him forever to sign in.\n\n\n\nAfter waiting nearly 10 minutes, I managed to convince him to cut his routine short and just inhale a couple of times while he was heading back to the car to get his laptop. It's 30 mins already.\n\nThen to top things off, I call Julie only to find out she's having intense Zoom issues and has to keep restarting Zoom.\n\n\n\nIt's another 15-minute wait before she's able to check her calendar. With almost an hour elapsed, I can guarantee the restaurant won't be taking any future calls from me.\n\n## There must be a better way\n\nAs we could see in the illustration above, all the different tasks that my colleagues were doing extended the timeline for the call. In the last example, I spent an entire hour trying to schedule a simple team lunch. Because there was a lineup at the gym, Kan had to wait, which meant I had to wait, which meant the restaurant customer service rep had to wait too.\n\nThis is of course isn't a great situation. The poor customer service rep at the restaurant shouldn't have had to wait an hour for me to book time at the restaurant. That's an hour they could be serving other customers, generating revenue, and working towards Employee of the Month. Instead, that rep has to explain to their boss how my colleague's diaper surgery caused a significant loss of revenue to the restaurant. Or, how a potential customer's inhaling took up 20 minutes of their time.\n\nThis sounds pretty odd, but every time we call a remote service without proper boundaries we're doing the same thing. Often, as part of fulfilling some requests, our systems call out to remote dependencies (caches, databases, queues, internal services, and external third parties). Every dependency has the potential to induce undue strain on our system. AWS could be having an outage, or another service we depend on could be experiencing some degradation. Without proper controls in place, one problem can dramatically impact our system's ava"])</script><script>self.__next_f.push([1,"ilability and reliability. In this world where microservices are quite common, it's typical to have a dizzying web of dependencies involved in the fulfillment of a simple request. Uncontrolled calls can cause significant contention and impact many of the services in the web.\n\nLet's look at how we improve.\n\n### Issue a deadline - timeouts, retries, backoff, and circuit breakers\n\n#### Timeouts\n\nThe first concept that's critical to containing the blast radius is timeouts. If we effectively gave everyone on the call a deadline -- say a minute or two, we can control the maximum length of the call. If I have three colleagues and I'm only willing to wait 2 minutes for each person to respond, then the call shouldn't go too far beyond 6 minutes. This would reduce the wildly varying possibility that the poor employee could be waiting anywhere between a few minutes to hours for me to complete my booking. This also contains the impact of any one person's tasks on our scheduling. If we have a cutoff then Nima's surgeries won't bubble up, causing me to wait, which pushes the restaurant employee further from that Employee of the Month Award.\n\nIn the age of microservices, transient failures and prolonged outages are quite common. One system may experience an influx of requests they weren't prepared to manage. Or, someone could have rolled out a faulty change at a third party that caused our requests to take longer, or fail. Any and all of these changes can cascade if they're not controlled. When we use timeouts, we dampen the impact of these stresses and strains. If the cache is experiencing some degradation, applying a timeout means we wait up to a specified maximum then we abandon the call. This ensures that a dependency failure (e.g. a cache failing) doesn't impact our consumers, or our consumers' consumers, and so on. By stopping the propagation we ensure this fault doesn't spread across the web of microservices and bring others down.\n\n#### Retries and Backoff\n\nAnother approach that's closely related to timeouts is Retries. If I called and realized that my colleague is busy, I could wait until the deadline was exceeded and then offer to call back in a few minutes. Chances are they might be less tied up in a few minutes. We can take a similar approach with our systems. If we reach out to a dependency and don't get a reply before the timeout, we can retry after a short delay. Typically that retry could be a few milliseconds. We can also set a maximum number of retries. Many dependency issues are transient. In such cases, retrying after a short delay increases the likelihood of a successful response. Placing a cap on the number of retries guarantees that prolonged issues don't cascade. If there's an outage, retries might likely fail too. Setting the right number of retries gives us the right balance between making our consumers wait, or failing early with an error.\n\nWhen we timeout and then delay for a fixed interval, it could compound the problem if our dependency is under heavy load. Imagine some dependency we're trying to reach is buckling under heavy load. If every caller is timing out and then retrying at a similar interval, the system could end up getting surges called a Thundering Herd. Effectively, everyone tries to get in, waits for a short delay, then everyone tries to get in again (roughly) at the same time. That complicates recovery for the target system. The system doesn't get to recover properly due to surges hitting it by clients retrying at similar intervals. Instead of using a fixed interval, we can randomize the delay before retrying. This helps breaks the synchronicity with other clients and effectively reduces the net load on the target server.\n\nTaking this strategy one step further, we can increase the time we're willing to wait with each retry. In particular, we can increase the maximum random delay exponentially. This allows us to provide more room for the target system to recover. If the problem is indeed transient, allowing the dependency some breathing room might be just what it needs to recover from whatever "])</script><script>self.__next_f.push([1,"degradation is occurring. This is widely known as Random Exponential Backoff. We're backing off after each request and calculating a random delay that we're willing to wait. On each retry, we increase the maximum possible time exponentially. This can really help in reducing the load on a target system. Often load from consumers compounds issues a system faces, so using Random Exponential Backoff helps to alleviate some of the load, allowing a system to recover.\n\n#### Circuit Breakers\n\nFinally, if I noticed I'd been unsuccessful in getting to one of my colleagues, I could make a decision to stop calling. Instead of continuously retrying, I could effectively postpone all attempts to call until after I've surpassed a fixed number of retries. This is another common strategy that helps to prevent failures from cascading. By making a firm decision to stop forwarding requests to a suboptimal dependency, we're preventing any degradation from impacting any of our consumers. Instead of continuing to wait, which means our consumers have to wait, we can just fail gracefully, and earlier with an appropriate error. Unfortunately, this does often mean loss of functionality for our customers, but the upside is the maintenance of availability and reliability. By choosing not to forward requests to a borked dependency, we are removing lots of potential contention on the network that could impact many other areas of our service. We're ensuring one bad apple doesn't spoil the entire bunch.\n\n## Conclusion\n\nAnd that's a wrap! I hope you enjoyed this post about cascading failures and how we can mitigate them in our systems. If we're not careful failures can cascade and have a sustained impact on our systems and impact our customers. The discussed strategies help us combat these issues while providing a graceful experience for our customers! Thanks for stopping by!\n\nI'd love to hear some of your stories about how these strategies worked for you! Let's chat!b:[\"$\",\"main\",null,{\"id\":\"main-content\",\"className\":\"min-h-screen bg-transparent\",\"children\":[[\"$\",\"$L17\",null,{\"publication\":{\"__typename\":\"Publication\",\"id\":\"625387b175dccb6f343206bb\",\"url\":\"https://jaywhy13.hashnode.dev\",\"canonicalURL\":\"https://jaywhy13.hashnode.dev\",\"urlPattern\":\"simple\",\"title\":\"Perspective Unspoken\",\"displayTitle\":null,\"hasBadges\":true,\"descriptionSEO\":\"Welcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for \",\"seo\":{\"title\":null,\"description\":null,\"__typename\":\"SEO\"},\"publicMembers\":{\"totalDocuments\":1,\"__typename\":\"PublicMembers\"},\"about\":{\"html\":\"\u003cp\u003eWelcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.\u003c/p\u003e\\n\",\"text\":\"Welcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.\\n\",\"__typename\":\"Content\"},\"features\":{\"proTeam\":{\"isEnabled\":false,\"__typename\":\"ProTeamFeature\"},\"newsletter\":{\"isEnabled\":true,\"__typename\":\"NewsletterFeature\"},\"viewCount\":{\"isEnabled\":false,\"__typename\":\"ViewCountFeature\"},\"readTime\":{\"isEnabled\":true,\"__typename\":\"ReadTimeFeature\"},\"textSelectionSharer\":{\"isEnabled\":true,\"__typename\":\"TextSelectionSharerFeature\"},\"gptBotCrawling\":{\"__typename\":\"GPTBotCrawlingFeature\",\"isEnabled\":false},\"headlessCMS\":{\"isEnabled\":false,\"__typename\":\"HeadlessCMSFeature\"},\"__typename\":\"PublicationFeatures\"},\"aeoSettings\":{\"isEnabled\":false,\"llmsTxtEnabled\":true,\"crawlers\":{\"gptBot\":false,\"claudeBot\":true,\"perplexityBot\":true,\"googleExtended\":true,\"ccBot\":true,\"__typename\":\"AeoCrawlerSettings\"},\"__typename\":\"PublicationAeoSettings\"},\"metaTags\":null,\"ogMetaData\":{\"image\":null,\"__typename\":\"OpenGraphMetaData\"},\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"name\":\"Jean-Mark Wright\",\"username\":\"jaywhy13\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1"])</script><script>self.__next_f.push([1,"fOZW2.jpeg\"},\"preferences\":{\"__typename\":\"PublicationPreferences\",\"logo\":null,\"darkMode\":{\"__typename\":\"DarkModePreferences\",\"logo\":null,\"enabled\":false},\"navbarItems\":[{\"__typename\":\"PublicationNavbarItem\",\"id\":\"66e232819d27b6c7f345ab6e\",\"label\":\"Calendar\",\"url\":\"https://calendar.google.com/calendar/u/0/appointments/AcZssZ3tWQd0YSITufTZaAitKeE6Ln2NBOouN5Qlop4=?gv=true\",\"type\":\"link\",\"series\":null,\"page\":null}],\"enabledPages\":{\"__typename\":\"EnabledPages\",\"badges\":true,\"newsletter\":false,\"members\":false},\"layout\":\"magazine\",\"disableFooterBranding\":false,\"isSubscriptionModalDisabled\":false},\"favicon\":null,\"headerColor\":\"#ff8484\",\"integrations\":{\"fbPixelID\":null,\"fathomSiteID\":null,\"fathomCustomDomainEnabled\":false,\"fathomCustomDomain\":null,\"hotjarSiteID\":null,\"matomoSiteID\":null,\"matomoURL\":null,\"gaTrackingID\":null,\"gTagManagerID\":null,\"plausibleAnalyticsEnabled\":false,\"wmPaymentPointer\":null,\"koalaPublicKey\":null,\"msClarityID\":null,\"openPanelClientId\":null,\"consentManagement\":{\"enabled\":false,\"version\":1,\"privacyPolicyURL\":null,\"__typename\":\"ConsentManagement\"},\"__typename\":\"PublicationIntegrations\"},\"imprintV2\":null,\"postsCount\":{\"totalDocuments\":19,\"__typename\":\"PostConnection\"},\"isTeam\":true,\"links\":{\"twitter\":\"https://twitter.com/kramnaej\",\"instagram\":null,\"github\":\"https://github.com/jaywhy13\",\"website\":null,\"hashnode\":\"https://hashnode.com/@jaywhy13\",\"youtube\":null,\"dailydev\":null,\"linkedin\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"mastodon\":null,\"facebook\":null,\"bluesky\":null,\"__typename\":\"PublicationLinks\"},\"domainInfo\":{\"__typename\":\"DomainInfo\",\"hashnodeSubdomain\":\"jaywhy13\",\"domain\":null,\"wwwPrefixedDomain\":null},\"redirectionRules\":[{\"source\":\"/two-inportant-lessons-i-learnt-rolling-out-a-new-platform-feature\",\"destination\":\"/two-important-lessons-i-learned-rolling-out-a-new-platform-feature\",\"type\":301,\"__typename\":\"RedirectionRule\"},{\"source\":\"/would-sherlock-use-traces-or-metrics-to-debug-your-application\",\"destination\":\"/3-reasons-traces-better-than-metrics-for-debugging-your-application\",\"type\":301,\"__typename\":\"RedirectionRule\"}],\"customRules\":[{\"type\":302,\"path\":\"/linkedin\",\"destination\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"__typename\":\"CustomRule\"}],\"totalRecommendedPublications\":0,\"allowContributorEdits\":false,\"widgets\":[],\"post\":{\"__typename\":\"Post\",\"id\":\"63e4e3f3b9f58f9b3c72ce21\",\"cuid\":\"cldx2cltm000508jt86dggtcn\",\"title\":\"Cascading failures and the impossibility of scheduling team lunches\",\"subtitle\":\"An illustrated guide on the importance of implementing timeouts and other features to increase system reliability\",\"slug\":\"cascading-failures-and-the-impossibility-of-scheduling-team-lunches\",\"brief\":\"Introduction\\nIn this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't...\",\"featured\":false,\"publishedAt\":\"2023-02-09T12:15:47.579Z\",\"updatedAt\":null,\"url\":\"https://jaywhy13.hashnode.dev/cascading-failures-and-the-impossibility-of-scheduling-team-lunches\",\"readTimeInMinutes\":9,\"views\":1568,\"reactionCount\":0,\"responseCount\":0,\"replyCount\":0,\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"name\":\"Jean-Mark Wright\",\"username\":\"jaywhy13\",\"deactivated\":false,\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"bio\":{\"html\":\"\u003cp\u003eJean-Mark is an articulate, customer-focused, visionary with extensive industry experience encompassing front and back end development in all phases of software development. He lives \\\"outside the box\\\" in pursuit of architectural elegance and relevant, tangible solutions. He is a strong believer that team empowerment fosters genius and is versed at working solo.\u003c/p\u003e\\n\",\"__typename\":\"Content\"},\"socialMediaLinks\":{\"website\":\"\",\"github\":\"https://github.com/jaywhy13\",\"twitter\":\"https://twitter.com/kramnaej\",\"facebook\":\"\",\"stackoverflow\":\"\",\"linkedin\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"__typename\":\"SocialMediaLinks\"}},\""])</script><script>self.__next_f.push([1,"coAuthors\":[],\"seo\":{\"title\":\"Cascading failures and the impossibility of scheduling team lunches\",\"description\":\"Introduction\\nIn this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't...\",\"__typename\":\"SEO\"},\"coverImage\":{\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg\",\"isPortrait\":false,\"attribution\":\"https://unsplash.com/@charlfolscher\",\"isAttributionHidden\":false,\"photographer\":\"Charl Folscher\",\"__typename\":\"CoverImage\"},\"content\":{\"html\":\"$18\",\"markdown\":\"$19\",\"__typename\":\"Content\"},\"preferences\":{\"pinnedToBlog\":false,\"disableComments\":false,\"stickCoverToBottom\":false,\"isDelisted\":false,\"__typename\":\"PostPreferences\"},\"series\":null,\"tags\":[{\"id\":\"58c920a729c05c8d79875621\",\"slug\":\"reliability\",\"name\":\"Reliability\",\"__typename\":\"Tag\"},{\"id\":\"576d0bde8acc52d77861cbb0\",\"slug\":\"timeout\",\"name\":\"timeout\",\"__typename\":\"Tag\"},{\"id\":\"63984be9d1e499a9b83abd42\",\"slug\":\"high-availability\",\"name\":\"high availability\",\"__typename\":\"Tag\"}],\"ogMetaData\":{\"image\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg\",\"__typename\":\"OpenGraphMetaData\"},\"canonicalUrl\":null,\"hasLatexInPost\":false,\"faq\":[],\"features\":{\"tableOfContents\":{\"isEnabled\":true,\"items\":[{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-0\",\"level\":2,\"slug\":\"introduction\",\"title\":\"Introduction\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-1\",\"level\":2,\"slug\":\"its-lunchtime\",\"title\":\"It's lunchtime!\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-2\",\"level\":2,\"slug\":\"there-must-be-a-better-way\",\"title\":\"There must be a better way\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-3\",\"level\":3,\"slug\":\"issue-a-deadline-timeouts-retries-backoff-and-circuit-breakers\",\"title\":\"Issue a deadline - timeouts, retries, backoff, and circuit breakers\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-4\",\"level\":4,\"slug\":\"timeouts\",\"title\":\"Timeouts\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-5\",\"level\":4,\"slug\":\"retries-and-backoff\",\"title\":\"Retries and Backoff\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-6\",\"level\":4,\"slug\":\"circuit-breakers\",\"title\":\"Circuit Breakers\",\"parentId\":null},{\"__typename\":\"TableOfContentsItem\",\"id\":\"toc-7\",\"level\":2,\"slug\":\"conclusion\",\"title\":\"Conclusion\",\"parentId\":null}],\"__typename\":\"TableOfContents\"},\"__typename\":\"PostFeatures\"}},\"posts\":{\"edges\":[{\"node\":{\"__typename\":\"Post\",\"id\":\"694a1633103f879a13892085\",\"title\":\"Instrumenting at 10 years per second\",\"slug\":\"instrumenting-at-10-years-per-second\",\"publishedAt\":\"2025-12-23T04:10:27.099Z\",\"cuid\":\"cmji2h6zf000102jx6k7x1e0n\",\"url\":\"https://jaywhy13.hashnode.dev/instrumenting-at-10-years-per-second\",\"subtitle\":\"How we made custom instrumentation blazing fast, simple, and data-centric\",\"brief\":\"Introduction\\nI've found a new love - custom instrumentation, and I can’t keep it to myself. We manually increased the custom instrumentation in one of our newer services and loved the impact. The instrumentation accelerated forensic investigations, r...\",\"readTimeInMinutes\":8,\"views\":276,\"tags\":[{\"id\":\"5f1df3b4f4016901885ccc96\",\"slug\":\"observability\",\"name\":\"observability\",\"__typename\":\"Tag\"},{\"id\":\"5fc2287b55b779631e1acc29\",\"slug\":\"instrumentation\",\"name\":\"instrumentation\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b94d67\",\"slug\":\"python\",\"name\":\"Python\",\"__typename\":\"Tag\"},{\"id\":\"56744723958ef13879b95342\",\"slug\":\"python3\",\"name\":\"Python 3\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/iR"])</script><script>self.__next_f.push([1,"nUeA04kUY/upload/473c705099a91403409785e9d27d20e9.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"67b2ab28c4ef92b38e5bdda0\",\"title\":\"On shipping smaller PRs\",\"slug\":\"on-shipping-smaller-prs\",\"publishedAt\":\"2025-02-17T03:21:12.558Z\",\"cuid\":\"cm78hnn8u000409l51hurfg9f\",\"url\":\"https://jaywhy13.hashnode.dev/on-shipping-smaller-prs\",\"subtitle\":\"Concrete examples for how to ship smaller PRs\",\"brief\":\"Introduction\\nIn my last I talked about the importance of shipping smaller PRs. In this article I’ll give some practical examples for how PRs can be split into smaller chunks. The examples aren’t organized into specific themes. Rather they’re arbitrar...\",\"readTimeInMinutes\":8,\"views\":83,\"tags\":[{\"id\":\"59c5f7f0ea46c0424704e8c1\",\"slug\":\"pull-requests\",\"name\":\"Pull Requests\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b949f9\",\"slug\":\"code-review\",\"name\":\"code review\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b94ad1\",\"slug\":\"software-development\",\"name\":\"software development\",\"__typename\":\"Tag\"},{\"id\":\"569d22c892921b8f79d35f68\",\"slug\":\"software-engineering\",\"name\":\"Software Engineering\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/o9oQaOGpLz0/upload/9e19fbfd378d65c448f7233b2a273b79.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"676b90a8cb76c2499dc67afb\",\"title\":\"Small PRs ship faster - QED\",\"slug\":\"small-prs-ship-faster-qed\",\"publishedAt\":\"2024-12-25T04:57:12.349Z\",\"cuid\":\"cm53fb3j1000309mb1rckcb01\",\"url\":\"https://jaywhy13.hashnode.dev/small-prs-ship-faster-qed\",\"subtitle\":\"My unrestrained passion for shipping smaller PRs constrained into a handful of benefits\",\"brief\":\"Introduction\\nWelcome to my very scientific paper where I propose that small PRs ship faster than big ones. I will construct my hypothesis with carefully crafted examples, concrete evidence and lots of words and sentences in general. I guarantee there...\",\"readTimeInMinutes\":5,\"views\":110,\"tags\":[{\"id\":\"56744721958ef13879b949f9\",\"slug\":\"code-review\",\"name\":\"code review\",\"__typename\":\"Tag\"},{\"id\":\"59c5f7f0ea46c0424704e8c1\",\"slug\":\"pull-requests\",\"name\":\"Pull Requests\",\"__typename\":\"Tag\"},{\"id\":\"5d9352eae235795f6eb6df65\",\"slug\":\"pull-request\",\"name\":\"pull request\",\"__typename\":\"Tag\"},{\"id\":\"56744723958ef13879b953b4\",\"slug\":\"review\",\"name\":\"review\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/JzMS3RLSZn8/upload/a18ee8f82d0d56644ab82f6c1bead8b4.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"671072daa4aac495b3ea5d5a\",\"title\":\"Order matters - making a compound index 50x faster\",\"slug\":\"order-matters-making-a-compound-index-50x-faster\",\"publishedAt\":\"2024-10-17T02:13:46.740Z\",\"cuid\":\"cm2co35h0000009ld0dg42agd\",\"url\":\"https://jaywhy13.hashnode.dev/order-matters-making-a-compound-index-50x-faster\",\"subtitle\":\"A story of how we made a compound index 50x faster by re-organizing the fields\",\"brief\":\"Introduction\\nToday I’ll talk about an endpoint that initially performed under 30ms, then crept up to ~500ms after a couple months. Our investigation revealed that a query, powered by a compound index was responsible for the elevated latency. We explo...\",\"readTimeInMinutes\":6,\"views\":1082,\"tags\":[{\"id\":\"56744722958ef13879b950eb\",\"slug\":\"databases\",\"name\":\"Databases\",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b949b5\",\"slug\":\"postgresql\",\"name\":\"PostgreSQL\",\"_"])</script><script>self.__next_f.push([1,"_typename\":\"Tag\"},{\"id\":\"56744722958ef13879b94f0b\",\"slug\":\"postgres\",\"name\":\"postgres\",\"__typename\":\"Tag\"},{\"id\":\"64308297311bf43ae8efe880\",\"slug\":\"postgresql-performance\",\"name\":\"Postgresql-performance \",\"__typename\":\"Tag\"},{\"id\":\"56744721958ef13879b94821\",\"slug\":\"optimization\",\"name\":\"optimization\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1726107899370/04b1ac6d-7825-4a05-b5d0-0be78a2f49d9.png\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"},{\"node\":{\"__typename\":\"Post\",\"id\":\"66e239b9dcb487d824fc1b6a\",\"title\":\"Dear Editor: We need better Database Observability\",\"slug\":\"dear-editor-we-need-better-database-observability\",\"publishedAt\":\"2024-09-12T00:45:45.852Z\",\"cuid\":\"cm0ykj5do000708lj0q3laq8c\",\"url\":\"https://jaywhy13.hashnode.dev/dear-editor-we-need-better-database-observability\",\"subtitle\":\"A call for better database observability, or awareness to a better reality\",\"brief\":\"Introduction\\nDear Editor, hear my plea. I pen this piece in earnest search of enlightenment, or the beginnings of a well needed change; we need better database observability! Specifically, more granular, and useful database spans. Coarse, high-level ...\",\"readTimeInMinutes\":8,\"views\":745,\"tags\":[{\"id\":\"5f1df3b4f4016901885ccc96\",\"slug\":\"observability\",\"name\":\"observability\",\"__typename\":\"Tag\"},{\"id\":\"56744722958ef13879b950eb\",\"slug\":\"databases\",\"name\":\"Databases\",\"__typename\":\"Tag\"}],\"author\":{\"__typename\":\"User\",\"id\":\"6253878075dccb6f343206b7\",\"username\":\"jaywhy13\",\"name\":\"Jean-Mark Wright\",\"profilePicture\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"followersCount\":40},\"coverImage\":{\"__typename\":\"CoverImage\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/NvFkYV2ngOk/upload/2f4cbc39f32355a334d9119af2dd086c.jpeg\",\"isPortrait\":false,\"isAttributionHidden\":false}},\"__typename\":\"PostEdge\"}],\"__typename\":\"PostConnection\"},\"redirectedPost\":null,\"staticPage\":null},\"posts\":[\"$b:props:children:0:props:publication:post\"],\"host\":\"jaywhy13.hashnode.dev\"}],\"$L1a\",\"$L1b\",null,\"$L1c\",\"$L1d\",null,\"$L1e\"]}]\n1f:I[62031,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"ReadingModeToggle\"]\n20:I[81356,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"Image\"]\n21:I[52619,[\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"8974\",\"static/chunks/app/page-cc18ae1a3a7a5ed8.js\"],\"\"]\n22:I[33884,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js"])</script><script>self.__next_f.push([1,"\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"Avatar\"]\n23:I[33884,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"AvatarImage\"]\n24:I[33884,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"AvatarFallback\"]\n27:I[51285,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"CommentsSection\"]\n28:I[20730,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"default\"]\n1a:[\"$\",\"section\",null,{\"className\":\"py-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"px-4 sm:px-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]\",\"children\":[\"$\",\"div\",null,{\"className\":\"grid gap-5 max-w-full\",\"children\":[\"$\",\"article\",null,{\"className\":\"w-full max-w-full rounded-xl border bg-card text-card-foreground\",\"children\":[[\"$\",\"div\",null,{\"className\":\"px-5 pt-12 pb-8 sm:px-8\",\"children\":[\"$\",\"header\",null,{\"className\":\"space-y-5\",\"children\":[\"$\",\"div\",null,{\"className\":\"space-y-4 text-center max-w-4xl mx-auto\",\"children\":[[\"$\",\"h1\",null,{\"className\":\"text-2xl sm:text-3xl md:text-4xl lg:text-5xl font-bold leading-tight tracking-tight text-balance text-foreground\",\"children\":\"Cascading failures and the impossibility of scheduling team lunches\"}],[\"$\",\"p\",null,{\"className\":\"text-lg sm:text-xl leading-normal font-normal text-muted-foreground text-balance\",\"children\":\"An illustrated guide on the importance of implementing timeouts and other features to increase system reliability\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center justify-center gap-3 text-sm text-muted-foreground\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-2\",\"children\":[[\"$\",\"span\",null,{\"children\":\"Published\"}],[\"$\",\"time\",null,{\"dateTime\":\"2023-02-09T12:15:47."])</script><script>self.__next_f.push([1,"579Z\",\"children\":\"February 9, 2023\"}]]}],[[\"$\",\"span\",null,{\"className\":\"text-muted-foreground\",\"children\":\"•\"}],[\"$\",\"span\",null,{\"children\":[9,\" min read\"]}]],[\"$\",\"span\",null,{\"className\":\"text-muted-foreground\",\"children\":\"•\"}],[\"$\",\"$L1f\",null,{\"slug\":\"cascading-failures-and-the-impossibility-of-scheduling-team-lunches\"}]]}]]}]}]}],[\"$\",\"div\",null,{\"className\":\"w-full\",\"children\":[\"$\",\"$L20\",null,{\"src\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg\",\"alt\":\"Cascading failures and the impossibility of scheduling team lunches\",\"width\":1200,\"height\":750,\"className\":\"w-full h-auto\",\"priority\":true}]}],[\"$\",\"div\",null,{\"className\":\"px-5 py-12 sm:px-8 space-y-8\",\"children\":[[\"$\",\"div\",null,{\"className\":\"max-w-2xl mx-auto\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center gap-5\",\"children\":[[\"$\",\"div\",\"6253878075dccb6f343206b7\",{\"className\":\"flex items-center gap-5\",\"children\":[false,[\"$\",\"div\",null,{\"className\":\"flex gap-3 items-start\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/@jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener\",\"children\":[\"$\",\"$L22\",null,{\"className\":\"h-10 w-10 border border-border transition-opacity hover:opacity-80\",\"children\":[[\"$\",\"$L23\",null,{\"src\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg?auto=compress,format\u0026format=webp\",\"alt\":\"Jean-Mark Wright\"}],[\"$\",\"$L24\",null,{\"className\":\"bg-muted text-muted-foreground font-semibold text-lg\",\"children\":\"J\"}]]}]}],[\"$\",\"div\",null,{\"className\":\"min-w-0\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-2\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/@jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener\",\"className\":\"font-medium text-foreground hover:text-primary transition-colors\",\"children\":\"Jean-Mark Wright\"}],[\"$\",\"div\",null,{\"className\":\"flex items-center gap-1.5 text-sm\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"https://twitter.com/kramnaej\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground hover:text-foreground transition-colors\",\"aria-label\":\"X (Twitter)\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-x-twitter\",\"aria-hidden\":true}]}],[\"$\",\"$L21\",null,{\"href\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground hover:text-foreground transition-colors\",\"aria-label\":\"LinkedIn\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-linkedin\",\"aria-hidden\":true}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"text-sm text-muted-foreground line-clamp-2\",\"dangerouslySetInnerHTML\":{\"__html\":\"\u003cp\u003eJean-Mark is an articulate, customer-focused, visionary with extensive industry experience encompassing front and back end development in all phases of software development. He lives \\\"outside the box\\\" in pursuit of architectural elegance and relevant, tangible solutions. He is a strong believer that team empowerment fosters genius and is versed at working solo.\u003c/p\u003e\\n\"}}]]}]]}]]}]]}]}],null,\"$L25\",false,\"$L26\"]}]]}]}]}]}]}]\n1b:[\"$\",\"section\",null,{\"className\":\"pb-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"px-4 sm:px-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-[90rem]\",\"children\":[\"$\",\"div\",null,{\"className\":\"max-w-4xl w-full mx-auto\",\"children\":[\"$\",\"div\",null,{\"className\":\"rounded-xl border bg-card text-card-foreground p-5 sm:p-8\",\"children\":[\"$\",\"$L27\",null,{\"host\":\"jaywhy13.hashnode.dev\",\"slug\":\"cascading-failures-and-the-impossibility-of-scheduling-team-lunches\",\"postUrl\":\"https://hashnode.com/posts/cascading-failures-and-the-impossibility-of-scheduling-team-lunches/63e4e3f3b9f58f9b3c72ce21\",\"initialEdges\":[],\"initialPageInfo\":{\"hasNextPage\":false,\"endCursor\":\"10\",\"totalDocuments\":0,\"__typename\":\"PageInfo\"},\"totalComments\":0}]}]}]}]}]}]\n1c:[\"$\",\"section\",null,{\"className\":\"pb-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"px-4 sm:px-8\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-full xl:mx-auto xl:max-w-[80rem] 2xl:max-w-["])</script><script>self.__next_f.push([1,"90rem]\",\"children\":[\"$\",\"div\",null,{\"className\":\"max-w-4xl w-full mx-auto\",\"children\":[\"$\",\"section\",null,{\"className\":\"space-y-6\",\"children\":[[\"$\",\"h2\",null,{\"className\":\"text-2xl font-bold tracking-tight text-balance\",\"children\":\"More from this blog\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-col gap-6\",\"children\":[[\"$\",\"$L28\",\"694a1633103f879a13892085\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:0:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"67b2ab28c4ef92b38e5bdda0\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:1:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"676b90a8cb76c2499dc67afb\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:2:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"671072daa4aac495b3ea5d5a\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:3:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"$L28\",\"66e239b9dcb487d824fc1b6a\",{\"post\":\"$b:props:children:0:props:publication:posts:edges:4:node\",\"showReadTime\":true,\"showViews\":false}],[\"$\",\"div\",\"author-card\",{\"ref\":\"$undefined\",\"className\":\"rounded-xl border bg-card text-card-foreground transition hover:border-slate-400 dark:hover:border-slate-600 p-5 flex flex-col\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex flex-col items-center text-center gap-5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-col items-center gap-3\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex h-16 w-16 shrink-0 items-center justify-center rounded-full bg-muted\",\"children\":[\"$\",\"i\",null,{\"className\":\"text-2xl fa-brands fa-hashnode text-foreground\",\"aria-hidden\":true}]}],[\"$\",\"div\",null,{\"className\":\"space-y-1\",\"children\":[[\"$\",\"p\",null,{\"className\":\"text-base font-semibold text-foreground leading-tight\",\"children\":\"Perspective Unspoken\"}],[\"$\",\"p\",null,{\"className\":\"text-sm text-muted-foreground\",\"children\":[19,\" posts\"]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"prose prose-sm dark:prose-invert max-w-xs mx-auto text-muted-foreground\",\"dangerouslySetInnerHTML\":{\"__html\":\"\u003cp\u003eWelcome to \\\"Perspective Unspoken\\\", here I talk about software development topics like software architecture, monitoring and observability and coaching tips for managers and employees.\u003c/p\u003e\\n\"}}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center justify-center gap-3\",\"children\":[[],[[\"$\",\"a\",\"Twitter\",{\"href\":\"https://twitter.com/kramnaej\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"Twitter\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-x-twitter\",\"aria-hidden\":true}]}],[\"$\",\"a\",\"GitHub\",{\"href\":\"https://github.com/jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"GitHub\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-github\",\"aria-hidden\":true}]}],[\"$\",\"a\",\"LinkedIn\",{\"href\":\"https://www.linkedin.com/in/jean-mark-wright/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"LinkedIn\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-linkedin\",\"aria-hidden\":true}]}],[\"$\",\"a\",\"Hashnode\",{\"href\":\"https://hashnode.com/@jaywhy13\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer nofollow ugc\",\"className\":\"text-muted-foreground transition-colors hover:text-foreground\",\"aria-label\":\"Hashnode\",\"children\":[\"$\",\"i\",null,{\"className\":\"fa-brands fa-hashnode\",\"aria-hidden\":true}]}]]]}]]}]}]]}]]}]}]}]}]}]\n29:T4dd,{\"@context\":\"https://schema.org\",\"@type\":\"Article\",\"url\":\"https://jaywhy13.hashnode.dev/cascading-failures-and-the-impossibility-of-scheduling-team-lunches\",\"mainEntityOfPage\":\"https://jaywhy13.hashnode.dev/cascading-failures-and-the-impossibility-of-scheduling-team-lunches\",\"headline\":\"Cascading failures and the impossibility of scheduling team lunches\",\"description\":\"Introduction\\nIn this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeou"])</script><script>self.__next_f.push([1,"ts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't...\",\"author\":{\"@type\":\"Person\",\"name\":\"Jean-Mark Wright\",\"url\":\"https://hashnode.com/@jaywhy13\",\"image\":\"https://cdn.hashnode.com/res/hashnode/image/upload/v1649641346570/vVg1fOZW2.jpeg\",\"sameAs\":\"https://x.com/kramnaej\"},\"publisher\":{\"@type\":\"Organization\",\"name\":\"Perspective Unspoken\",\"url\":\"https://jaywhy13.hashnode.dev\"},\"datePublished\":\"2023-02-09T12:15:47.579Z\",\"dateModified\":\"2023-02-09T12:15:47.579Z\",\"image\":{\"@type\":\"ImageObject\",\"url\":\"https://cdn.hashnode.com/res/hashnode/image/stock/unsplash/8pwPnveOEiE/upload/e86de783b0adba18854269892cc40f11.jpeg\"},\"keywords\":\"Reliability, timeout, high availability\"}1d:[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"$29\"}}]\n2a:I[78655,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"TableOfContents\"]\n2b:T31cb,\u003ch2 id=\"heading-introduction\"\u003eIntroduction\u003c/h2\u003e\n\u003cp\u003eIn this article, I'm going to talk about the impact of cascading failures on a system and how we can mitigate the impact using timeouts, retries and circuit breakers. When a failure happens in one of your systems or components, you don't want that failure propagating, or cascading to other areas. You must control the blast radius, or everything will go up in flames. Ideally, you want to ensure that your components absorb shocks and strains and don't provide a pathway for propagation. One uncontained fault can jeopardize your system's availability and reliability. We can use timeouts, circuit breakers, and other techniques to reduce the impact.\u003c/p\u003e\n\u003ch2 id=\"heading-its-lunchtime\"\u003eIt's lunchtime!\u003c/h2\u003e\n\u003cp\u003eTo illustrate the problem, we'll consider the task of scheduling team lunch for a rather busy set of colleagues. Let's assume everybody is working remotely and tends to have a life, which gets in the way of team initiatives. This makes scheduling a rather interesting adventure. Nobody pays attention to the polls in Slack to decide on a time for lunch, so we have to resort to calling everyone manually. For our consideration, let's assume I called the restaurant and was fortunate enough to get a patient representative who waited while I called my other team members to book.\u003c/p\u003e\n\u003cp\u003eHave a look at the diagram below. Hopefully, it looks familiar. It's a trace graph! Each box represents some action that's happening. The first box represents the duration of my call to the restaurant. The boxes in the next line represent each co-worker I'm calling in an attempt to schedule the team lunch. Each box below represents the life commitments that make scheduling a time-consuming affair. Let's take a closer look!\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1674532131705/8577af52-25a5-4e67-aaec-029cd19d8ee7.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eSo I called the restaurant and asked them to hold while I connected with my colleagues and confirmed. First up, I got a Pakistani colleague we'll call Nima. He has a beautiful baby girl. When I called he just happened to be in the middle of a diaper change. Yikes. After surgically removing the diaper and installing a clean one, it took him a few more minutes to get back to his computer and check his calendar. He's free! Next up, Kan, a health-conscious teammate who faithfully and consistently goes to the gym once or twice every couple of months. Just my luck! Gym time just happens to be now. I had to wait until he signed in and did his intensive workout session consisting of inhaling and stretching. 15 minutes in, he's able to get ba"])</script><script>self.__next_f.push([1,"ck to his computer and check his schedule, he's free! Finally, let's check in with Julie. It's almost 20 minutes and I can hear the receptionist on the line getting impatient. She's having issues getting Zoom started, and she's about to head into a quick meeting. Close to the half-hour mark, Julie confirms that she too is free! Overjoyed, the receptionist books us in and terminates the call.\u003c/p\u003e\n\u003cp\u003ePhew... that only took 30 minutes. The receptionist had to wait for me to connect with all my colleagues. All of them had different things on their plate and were unable to respond immediately. In many cases, the stuff on their plate (not talking about Nima's plate... obviously) also took time. Now, let's look at a rather extreme example, then see what we can learn from this illustration.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1674533284241/8edab660-a06f-4059-8383-1145da6c39fa.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eSo this time, let's say Nima's surgery didn't go as planned and he decided to take a shower.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1674533523361/7d8cc3ea-3cfe-4a0f-932f-0ac0a2a1be43.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eI had to wait 15 minutes just to have him check his calendar and confirm his availability. Then, let's say Kan ran into a long line at the gym, so it took him forever to sign in.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1674533591973/836f51ef-2bd6-4720-8cb8-e8f87c455216.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eAfter waiting nearly 10 minutes, I managed to convince him to cut his routine short and just inhale a couple of times while he was heading back to the car to get his laptop. It's 30 mins already.\u003c/p\u003e\n\u003cp\u003eThen to top things off, I call Julie only to find out she's having intense Zoom issues and has to keep restarting Zoom.\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"https://cdn.hashnode.com/res/hashnode/image/upload/v1674533661580/9ccc409e-5462-4a70-b54a-e59f1a98cd08.png\" alt class=\"image--center mx-auto\" /\u003e\u003c/p\u003e\n\u003cp\u003eIt's another 15-minute wait before she's able to check her calendar. With almost an hour elapsed, I can guarantee the restaurant won't be taking any future calls from me.\u003c/p\u003e\n\u003ch2 id=\"heading-there-must-be-a-better-way\"\u003eThere must be a better way\u003c/h2\u003e\n\u003cp\u003eAs we could see in the illustration above, all the different tasks that my colleagues were doing extended the timeline for the call. In the last example, I spent an entire hour trying to schedule a simple team lunch. Because there was a lineup at the gym, Kan had to wait, which meant I had to wait, which meant the restaurant customer service rep had to wait too.\u003c/p\u003e\n\u003cp\u003eThis is of course isn't a great situation. The poor customer service rep at the restaurant shouldn't have had to wait an hour for me to book time at the restaurant. That's an hour they could be serving other customers, generating revenue, and working towards Employee of the Month. Instead, that rep has to explain to their boss how my colleague's diaper surgery caused a significant loss of revenue to the restaurant. Or, how a potential customer's inhaling took up 20 minutes of their time.\u003c/p\u003e\n\u003cp\u003eThis sounds pretty odd, but every time we call a remote service without proper boundaries we're doing the same thing. Often, as part of fulfilling some requests, our systems call out to remote dependencies (caches, databases, queues, internal services, and external third parties). Every dependency has the potential to induce undue strain on our system. AWS could be having an outage, or another service we depend on could be experiencing some degradation. Without proper controls in place, one problem can dramatically impact our system's availability and reliability. In this world where microservices are quite common, it's typical to have a dizzying web of dependencies involved in the fulfillment of a simple request. Uncontrolled calls can cause significant contention and impact many of the services in the web.\u003c/p\u003e\n\u003cp\u003eLet's look at how we improve.\u003c/p\u003e\n\u003ch3 id=\"heading-issue-a-deadline-timeouts-re"])</script><script>self.__next_f.push([1,"tries-backoff-and-circuit-breakers\"\u003eIssue a deadline - timeouts, retries, backoff, and circuit breakers\u003c/h3\u003e\n\u003ch4 id=\"heading-timeouts\"\u003eTimeouts\u003c/h4\u003e\n\u003cp\u003eThe first concept that's critical to containing the blast radius is timeouts. If we effectively gave everyone on the call a deadline -- say a minute or two, we can control the maximum length of the call. If I have three colleagues and I'm only willing to wait 2 minutes for each person to respond, then the call shouldn't go too far beyond 6 minutes. This would reduce the wildly varying possibility that the poor employee could be waiting anywhere between a few minutes to hours for me to complete my booking. This also contains the impact of any one person's tasks on our scheduling. If we have a cutoff then Nima's surgeries won't bubble up, causing me to wait, which pushes the restaurant employee further from that Employee of the Month Award.\u003c/p\u003e\n\u003cp\u003eIn the age of microservices, transient failures and prolonged outages are quite common. One system may experience an influx of requests they weren't prepared to manage. Or, someone could have rolled out a faulty change at a third party that caused our requests to take longer, or fail. Any and all of these changes can cascade if they're not controlled. When we use timeouts, we dampen the impact of these stresses and strains. If the cache is experiencing some degradation, applying a timeout means we wait up to a specified maximum then we abandon the call. This ensures that a dependency failure (e.g. a cache failing) doesn't impact our consumers, or our consumers' consumers, and so on. By stopping the propagation we ensure this fault doesn't spread across the web of microservices and bring others down.\u003c/p\u003e\n\u003ch4 id=\"heading-retries-and-backoff\"\u003eRetries and Backoff\u003c/h4\u003e\n\u003cp\u003eAnother approach that's closely related to timeouts is Retries. If I called and realized that my colleague is busy, I could wait until the deadline was exceeded and then offer to call back in a few minutes. Chances are they might be less tied up in a few minutes. We can take a similar approach with our systems. If we reach out to a dependency and don't get a reply before the timeout, we can retry after a short delay. Typically that retry could be a few milliseconds. We can also set a maximum number of retries. Many dependency issues are transient. In such cases, retrying after a short delay increases the likelihood of a successful response. Placing a cap on the number of retries guarantees that prolonged issues don't cascade. If there's an outage, retries might likely fail too. Setting the right number of retries gives us the right balance between making our consumers wait, or failing early with an error.\u003c/p\u003e\n\u003cp\u003eWhen we timeout and then delay for a fixed interval, it could compound the problem if our dependency is under heavy load. Imagine some dependency we're trying to reach is buckling under heavy load. If every caller is timing out and then retrying at a similar interval, the system could end up getting surges called a Thundering Herd. Effectively, everyone tries to get in, waits for a short delay, then everyone tries to get in again (roughly) at the same time. That complicates recovery for the target system. The system doesn't get to recover properly due to surges hitting it by clients retrying at similar intervals. Instead of using a fixed interval, we can randomize the delay before retrying. This helps breaks the synchronicity with other clients and effectively reduces the net load on the target server.\u003c/p\u003e\n\u003cp\u003eTaking this strategy one step further, we can increase the time we're willing to wait with each retry. In particular, we can increase the maximum random delay exponentially. This allows us to provide more room for the target system to recover. If the problem is indeed transient, allowing the dependency some breathing room might be just what it needs to recover from whatever degradation is occurring. This is widely known as Random Exponential Backoff. We're backing off after each request and calculating a random delay that we're willing to wait. On each re"])</script><script>self.__next_f.push([1,"try, we increase the maximum possible time exponentially. This can really help in reducing the load on a target system. Often load from consumers compounds issues a system faces, so using Random Exponential Backoff helps to alleviate some of the load, allowing a system to recover.\u003c/p\u003e\n\u003ch4 id=\"heading-circuit-breakers\"\u003eCircuit Breakers\u003c/h4\u003e\n\u003cp\u003eFinally, if I noticed I'd been unsuccessful in getting to one of my colleagues, I could make a decision to stop calling. Instead of continuously retrying, I could effectively postpone all attempts to call until after I've surpassed a fixed number of retries. This is another common strategy that helps to prevent failures from cascading. By making a firm decision to stop forwarding requests to a suboptimal dependency, we're preventing any degradation from impacting any of our consumers. Instead of continuing to wait, which means our consumers have to wait, we can just fail gracefully, and earlier with an appropriate error. Unfortunately, this does often mean loss of functionality for our customers, but the upside is the maintenance of availability and reliability. By choosing not to forward requests to a borked dependency, we are removing lots of potential contention on the network that could impact many other areas of our service. We're ensuring one bad apple doesn't spoil the entire bunch.\u003c/p\u003e\n\u003ch2 id=\"heading-conclusion\"\u003eConclusion\u003c/h2\u003e\n\u003cp\u003eAnd that's a wrap! I hope you enjoyed this post about cascading failures and how we can mitigate them in our systems. If we're not careful failures can cascade and have a sustained impact on our systems and impact our customers. The discussed strategies help us combat these issues while providing a graceful experience for our customers! Thanks for stopping by!\u003c/p\u003e\n\u003cp\u003eI'd love to hear some of your stories about how these strategies worked for you! Let's chat!\u003c/p\u003e\n25:[\"$\",\"div\",null,{\"className\":\"relative\",\"children\":[[\"$\",\"aside\",null,{\"className\":\"hidden xl:block absolute top-0 left-0 h-full\",\"children\":[\"$\",\"nav\",null,{\"className\":\"sticky top-22 w-44\",\"children\":[\"$\",\"$L2a\",null,{\"headings\":[{\"id\":\"heading-introduction\",\"text\":\"Introduction\",\"level\":2},{\"id\":\"heading-its-lunchtime\",\"text\":\"It's lunchtime!\",\"level\":2},{\"id\":\"heading-there-must-be-a-better-way\",\"text\":\"There must be a better way\",\"level\":2},{\"id\":\"heading-issue-a-deadline-timeouts-retries-backoff-and-circuit-breakers\",\"text\":\"Issue a deadline - timeouts, retries, backoff, and circuit breakers\",\"level\":3},{\"id\":\"heading-timeouts\",\"text\":\"Timeouts\",\"level\":4},{\"id\":\"heading-retries-and-backoff\",\"text\":\"Retries and Backoff\",\"level\":4},{\"id\":\"heading-circuit-breakers\",\"text\":\"Circuit Breakers\",\"level\":4},{\"id\":\"heading-conclusion\",\"text\":\"Conclusion\",\"level\":2}]}]}]}],[\"$\",\"div\",null,{\"className\":\"max-w-2xl mx-auto min-w-0 contain-[inline-size]\",\"children\":[\"$\",\"div\",null,{\"className\":\"prose dark:prose-invert prose-pre:bg-foreground prose-pre:text-background prose-pre:border prose-pre:border-border prose-pre:rounded prose-pre:px-5 prose-pre:py-4 dark:prose-pre:bg-muted dark:prose-pre:text-foreground dark:prose-pre:border-border prose-pre:overflow-x-auto max-w-none text-base sm:text-lg [\u0026\u003ediv\u003ep:first-child]:mt-0 [\u0026\u003ediv\u003ep:first-child]:pt-0 min-w-0 wrap-break-word [\u0026_a]:break-all **:max-w-full\",\"children\":[[],[\"$\",\"div\",null,{\"suppressHydrationWarning\":true,\"dangerouslySetInnerHTML\":{\"__html\":\"$2b\"}}],[],\"$L2c\",\"$L2d\",\"$L2e\",\"$L2f\",\"$L30\"]}]}]]}]\n26:[\"$\",\"div\",null,{\"className\":\"max-w-2xl mx-auto border-t border-border pt-8 space-y-5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center gap-2\",\"children\":[[\"$\",\"$L21\",\"58c920a729c05c8d79875621\",{\"href\":\"/tag/reliability\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"reliability\"]}],[\"$\",\"$L21\",\"576d0bde8acc52d77861cbb0\",{\"href\":\"/tag/timeout\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:borde"])</script><script>self.__next_f.push([1,"r-primary hover:text-primary\",\"children\":[\"#\",\"timeout\"]}],[\"$\",\"$L21\",\"63984be9d1e499a9b83abd42\",{\"href\":\"/tag/high-availability\",\"className\":\"inline-flex items-center gap-1.5 rounded-full border border-border px-3 py-1 text-sm text-muted-foreground transition hover:border-primary hover:text-primary\",\"children\":[\"#\",\"high-availability\"]}]]}],false]}]\n31:I[32478,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"CodeHighlighter\"]\n32:I[7714,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"EmbedHandler\"]\n33:I[54960,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"ImageZoom\"]\n34:I[6671,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"MathJaxHandler\"]\n35:I[91337,[\"590\",\"static/chunks/0dbeb660-9e82e99813552576.js\",\"4554\",\"static/chunks/4554-984b480a9db5f3fe.js\",\"7563\",\"static/chunks/7563-4dfa0994a9b0e37c.js\",\"5001\",\"static/chunks/5001-fae0bebdd93f95e3.js\",\"2423\",\"static/chunks/2423-c8b1388240338d7d.js\",\"9216\",\"static/chunks/9216-9272709ba40c1cfd.js\",\"1545\",\"static/chunks/1545-7abf96433ff6553f.js\",\"3091\",\"static/chunks/3091-bf1537ff6571a1bd.js\",\"652\",\"static/chunks/652-f40dcb480b4e356f.js\",\"730\",\"static/chunks/730-8191af9e9424b148.js\",\"3048\",\"static/chunks/app/%5B...slug%5D/page-ce41bf29cdfb993a.js\"],\"MermaidHandler\"]\n2c:[\"$\",\"$L31\",null,{}]\n2d:[\"$\",\"$L32\",null,{}]\n2e:[\"$\",\"$L33\",null,{}]\n2f:[\"$\",\"$L34\",null,{\"hasLatex\":false}]\n30:[\"$\",\"$L35\",null,{}]\n36:Tc98,M268.379 39q-6.322 0-10.35-4.108-4.002-4.134-4.002-10.483 0-6.296 4.002-10.43 4-4.135 10.163-4.135 6.028 0 9.817 3.734 3.814 3.735 3.814 9.497v2.8h-24.248v-4.054h17.633q-.187-2.747-2.054-4.508-1.868-1.76-4.962-1.76-3.28-.001-5.388 2.374-2.108 2.373-2.108 6.375 0 4.188 2.348 6.695 2.347 2.481 5.895 2.481 1.68 0 2.934-.453a8.5 8.5 0 0 0 2.321-1.307q1.041-.854 2.268-2.614l4.694 2.96q-1.707 2.748-3.494 4.135a12.7 12.7 0 0 1-3.975 2.081q-2.187.72-5.308.72m-33.496 0q-5.575 0-9.283-4.028-3.681-4.029-3.681-10.484 0-6.43 3.761-10.51 3.76-4.108 9.47-4.108 2.8 0 5.175 1.2 2.4 1.174 3.628 3.148l.026-.08V0h7.016v38.2h-6.722v-4.722l.64.8-.694.133q-1.28 2.108-3.788 3.361Q237.924 39 234.883 39m1.627-6.002q3.308 0 5.495-2.428t2.188-6"])</script><script>self.__next_f.push([1,".135q0-3.762-2.214-6.109-2.214-2.374-5.469-2.374-3.28 0-5.522 2.347-2.214 2.348-2.214 6.163 0 3.76 2.214 6.162 2.214 2.374 5.522 2.374M205.811 39q-6.376 0-10.537-4.161-4.161-4.188-4.161-10.404 0-6.321 4.161-10.456 4.188-4.135 10.537-4.135 6.348 0 10.537 4.161 4.188 4.161 4.188 10.43 0 6.216-4.188 10.404Q212.186 39 205.811 39m0-6.162q3.307 0 5.575-2.374 2.267-2.374 2.267-6.055 0-3.707-2.294-6.056-2.267-2.347-5.548-2.347-3.308 0-5.575 2.374-2.268 2.348-2.268 6.029t2.241 6.055q2.268 2.374 5.602 2.374M162.586 38.2V10.644h6.802v5.015l-.8-.987h.906q1.308-2.187 3.762-3.494 2.48-1.335 5.228-1.334 4.615 0 7.256 2.8 2.667 2.776 2.667 7.603V38.2h-7.069V21.794q0-2.854-1.387-4.401t-4.134-1.547q-2.614 0-4.402 1.974-1.76 1.974-1.76 4.881V38.2zm-30.181 0V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.069V21.794q0-2.827-1.387-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.814 2.027-1.814 4.962v15.391zm-14.416.8q-2.882 0-5.015-.64-2.108-.614-3.761-1.84-1.654-1.228-2.668-2.695l-.987-1.467 4.695-3.174.773 1.067a9.1 9.1 0 0 0 1.788 1.814 8.2 8.2 0 0 0 2.32 1.2q1.308.4 3.068.4 2.161 0 3.441-.8 1.308-.8 1.307-2.294 0-1.36-1.147-2.108-1.12-.747-5.068-1.654-5.335-1.253-7.656-3.334-2.32-2.107-2.321-5.362 0-3.76 3.148-6.002 3.148-2.267 7.923-2.267 2.454 0 4.348.507t3.388 1.493q1.493.988 2.454 2.188l.987 1.2-4.482 3.255-.667-.774a7.4 7.4 0 0 0-1.627-1.387 6.4 6.4 0 0 0-2.054-.88 9.5 9.5 0 0 0-2.401-.294q-1.974 0-3.201.8-1.2.774-1.2 1.921 0 1.254 1.12 2.027 1.147.748 5.229 1.681 5.334 1.227 7.629 3.308 2.294 2.055 2.294 5.308 0 4.029-3.228 6.43Q123.217 39 117.989 39M87.61 39q-4.508 0-7.335-2.454-2.828-2.481-2.828-6.482 0-4.188 3.254-6.669t8.483-2.48q2.082 0 4.028.426 1.974.427 3.361 1.12v-1.6q0-2.588-1.813-4.082-1.788-1.493-4.615-1.493-1.628 0-2.801.373a8.7 8.7 0 0 0-2.188.987q-.987.614-1.84 1.414l-.854.8-4.081-3.788 1.307-1.173a18.7 18.7 0 0 1 2.934-2.108q1.601-.933 3.601-1.44t4.589-.507q5.868 0 9.229 2.88 3.361 2.882 3.361 8.084V38.2h-6.749v-5.922l1.654 2.187h-1.76q-1.308 2.055-3.655 3.308Q90.545 39 87.611 39m1.814-4.988q3.148 0 5.149-2.187 2-2.215 2-5.309v-.08q-1.12-.693-2.72-1.094a13 13 0 0 0-3.388-.426q-2.828 0-4.562 1.2-1.734 1.173-1.734 3.521 0 2.028 1.468 3.201t3.787 1.174M49 38.2V0h6.962v14.698h.134q1.2-2.187 3.574-3.521 2.375-1.335 5.229-1.334 4.615 0 7.255 2.828 2.668 2.801 2.668 7.522V38.2h-7.07V21.794q0-2.827-1.386-4.402-1.387-1.573-4.055-1.573-2.587 0-4.428 2.027-1.815 2.027-1.814 4.962v15.391z1e:[\"$\",\"footer\",null,{\"className\":\"px-4 pt-8 pb-20 sm:px-8 text-muted-foreground\",\"aria-label\":\"Footer\",\"children\":[\"$\",\"div\",null,{\"className\":\"w-full max-w-4xl mx-auto\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex flex-col items-center gap-4 text-center text-sm\",\"children\":[false,[\"$\",\"p\",null,{\"className\":\"font-medium\",\"children\":[[\"$\",\"time\",null,{\"dateTime\":\"2026\",\"children\":[[\"$\",\"span\",null,{\"className\":\"font-sans\",\"children\":\"©\"}],\" \",2026]}],\" \",\"Perspective Unspoken\"]}],[\"$\",\"nav\",null,{\"aria-label\":\"Footer navigation\",\"children\":[\"$\",\"ul\",null,{\"className\":\"flex flex-wrap justify-center gap-4 text-sm tracking-wide\",\"children\":[[\"$\",\"li\",\"/archive\",{\"children\":[\"$\",\"$L21\",null,{\"href\":\"/archive\",\"className\":\"transition hover:text-foreground\",\"target\":\"$undefined\",\"rel\":\"$undefined\",\"children\":\"Archive\"}]}],[\"$\",\"li\",\"https://hashnode.com/privacy-policy\",{\"children\":[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/privacy-policy\",\"className\":\"transition hover:text-foreground\",\"target\":\"_blank\",\"rel\":\"noopener\",\"children\":\"Privacy\"}]}],[\"$\",\"li\",\"https://hashnode.com/terms\",{\"children\":[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/terms\",\"className\":\"transition hover:text-foreground\",\"target\":\"_blank\",\"rel\":\"noopener\",\"children\":\"Terms\"}]}]]}]}],[\"$\",\"div\",null,{\"className\":\"pt-8 flex justify-center gap-6\",\"children\":[[\"$\",\"$L21\",null,{\"href\":\"/sitemap.xml\",\"target\":\"_blank\",\"rel\":\"noopener\",\"className\":\"flex items-center gap-2 text-sm transition hover:text-foreground\",\"title\":\"Sitemap\",\"children\":[[\"$\",\"i\",null,{\"className\":\"fa-regular fa-sitemap\"}],[\"$\",\"span\""])</script><script>self.__next_f.push([1,",null,{\"children\":\"Sitemap\"}]]}],[\"$\",\"$L21\",null,{\"href\":\"/rss.xml\",\"target\":\"_blank\",\"rel\":\"noopener\",\"className\":\"flex items-center gap-2 text-sm transition hover:text-foreground\",\"title\":\"RSS Feed\",\"children\":[[\"$\",\"i\",null,{\"className\":\"fa-regular fa-rss\"}],[\"$\",\"span\",null,{\"children\":\"RSS\"}]]}]]}],[\"$\",\"div\",null,{\"className\":\"pt-10 text-center\",\"children\":[\"$\",\"$L21\",null,{\"href\":\"https://hashnode.com/?utm_source=https%3A%2F%2Fjaywhy13.hashnode.dev\u0026utm_medium=referral\u0026utm_campaign=blog_footer_logo\u0026utm_content=logo\",\"target\":\"_blank\",\"rel\":\"noopener\",\"aria-label\":\"Visit Hashnode (opens in a new tab)\",\"children\":[\"$\",\"svg\",null,{\"fill\":\"none\",\"viewBox\":\"0 0 282 39\",\"aria-hidden\":\"true\",\"className\":\"h-6 w-auto min-w-[9.5rem] text-muted-foreground transition hover:text-foreground\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"currentColor\",\"d\":\"M2.68 13.032c-3.573 3.505-3.573 9.363 0 12.936L13.032 36.32c3.505 3.573 9.363 3.573 12.936 0L36.32 25.968c3.573-3.573 3.573-9.431 0-12.936L25.968 2.68c-3.573-3.573-9.431-3.573-12.936 0zm12.211 1.935c2.507-2.521 6.582-2.544 9.104-.038s2.544 6.582.038 9.104-6.582 2.544-9.104.038-2.544-6.582-.038-9.104\"}],[\"$\",\"path\",null,{\"fill\":\"currentColor\",\"d\":\"$36\"}]]}],\"className\":\"inline-flex items-center justify-center gap-2 whitespace-nowrap font-medium transition-colors focus-visible:outline-hidden focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [\u0026_svg]:pointer-events-none [\u0026_svg]:size-4 [\u0026_svg]:shrink-0 hover:bg-accent hover:text-accent-foreground h-10 rounded-md text-xl px-5 py-4\",\"ref\":null}]}]]}]}]}]\n"])</script></body></html> |