Files
nexus/sreweekly/articles/245/07-why-you-should-frequently-turn-down-30-of-canary-instances.html
2026-09-12 17:23:01 +08:00

491 lines
120 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8" />
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
<link rel="preconnect" href="https://substackcdn.com" />
<title data-rh="true">Why you should frequently turn down ~30% of canary instances</title>
<meta data-rh="true" name="theme-color" content="#fafafa"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Why you should frequently turn down ~30% of canary instances"/><meta data-rh="true" name="twitter:title" content="Why you should frequently turn down ~30% of canary instances"/><meta data-rh="true" name="description" content="The most effective (if scary) way to understand how your stateless service operates under load"/><meta data-rh="true" property="og:description" content="The most effective (if scary) way to understand how your stateless service operates under load"/><meta data-rh="true" name="twitter:description" content="The most effective (if scary) way to understand how your stateless service operates under load"/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!UM-M!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Freliability.substack.com%2Ftwitter%2Fsubscribe-card.jpg%3Fv%3D-795849870%26version%3D9"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!UM-M!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Freliability.substack.com%2Ftwitter%2Fsubscribe-card.jpg%3Fv%3D-795849870%26version%3D9"/><meta data-rh="true" name="twitter:card" content="summary"/><meta data-rh="true" property="article:modified_time" content="2023-01-26T13:45:13.356Z"/>
<style>
@layer legacy, pencraftReset, pencraft;
</style>
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.cee9fbee.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/86118.c25b9f67.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/76768.f15fc605.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/41932.97d2663c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/32552.313415d5.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
<meta name="author" content="Utsav Shah" />
<meta property="og:url" content="https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down" />
<link rel="canonical" href="https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down" />
<meta name="google-site-verification" content="IORiP7_fax0ESrUYObrVUA0QF8qgq2BzPsISmpFzgfg" />
<link rel="shortcut icon" href="https://substackcdn.com/image/fetch/$s_!HSk1!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Ffavicon.ico">
<link rel="icon" type="image/png" sizes="16x16" href="https://substackcdn.com/image/fetch/$s_!GXS9!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Ffavicon-16x16.png">
<link rel="icon" type="image/png" sizes="32x32" href="https://substackcdn.com/image/fetch/$s_!Wf00!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Ffavicon-32x32.png">
<link rel="apple-touch-icon" sizes="57x57" href="https://substackcdn.com/image/fetch/$s_!brnx!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="https://substackcdn.com/image/fetch/$s_!KKA0!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="https://substackcdn.com/image/fetch/$s_!DlLP!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="https://substackcdn.com/image/fetch/$s_!QdaO!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="https://substackcdn.com/image/fetch/$s_!pc-G!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="https://substackcdn.com/image/fetch/$s_!ItDq!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="https://substackcdn.com/image/fetch/$s_!UsOA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="https://substackcdn.com/image/fetch/$s_!kiLr!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-152x152.png">
<link rel="apple-touch-icon" sizes="167x167" href="https://substackcdn.com/image/fetch/$s_!nSUQ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-167x167.png">
<link rel="apple-touch-icon" sizes="180x180" href="https://substackcdn.com/image/fetch/$s_!vQUt!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-180x180.png">
<link rel="apple-touch-icon" sizes="1024x1024" href="https://substackcdn.com/image/fetch/$s_!K39t!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-1024x1024.png">
<link rel="alternate" type="application/rss+xml" href="/feed" title="Software at Scale"/>
<style data-pub-theme-prefix>:root{--color_theme_bg_pop:#1a2e05;--background_pop:#1a2e05;--color_theme_bg_web:#fafafa;--cover_bg_color:#fafafa;--cover_bg_color_secondary:#ebebeb;--background_pop_darken:#0d1703;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#0d1703;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(26, 46, 5, 0.2);--color_theme_bg_pop_30:rgba(26, 46, 5, 0.3);--print_pop:#1a2e05;--color_theme_accent:#1a2e05;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#1a2e05;--font_family_headings_preset:'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';--font_weight_headings_preset:900;--font_family_body_preset:'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';--font_weight_body_preset:400;--font_preset_heading:heavy_sans;--font_preset_body:sans;--home_hero:magaziney;--home_posts:list;--web_bg_color:#fafafa;--background_contrast_1:#ebebeb;--background_contrast_2:#d8d8d8;--background_contrast_3:#b3b3b3;--background_contrast_4:#909090;--background_contrast_5:#4f4f4f;--color_theme_bg_contrast_1:#ebebeb;--color_theme_bg_contrast_2:#d8d8d8;--color_theme_bg_contrast_3:#b3b3b3;--color_theme_bg_contrast_4:#909090;--color_theme_bg_contrast_5:#4f4f4f;--color_theme_bg_elevated:#fafafa;--color_theme_bg_elevated_secondary:#ebebeb;--color_theme_bg_elevated_tertiary:#d8d8d8;--color_theme_detail:#e1e1e1;--background_contrast_pop:rgba(26, 46, 5, 0.4);--color_theme_bg_contrast_pop:rgba(26, 46, 5, 0.4);--theme_bg_is_dark:0;--print_on_web_bg_color:#363737;--print_secondary_on_web_bg_color:#848585;--background_pop_rgb:26, 46, 5;--color_theme_bg_pop_rgb:26, 46, 5;--color_theme_accent_rgb:26, 46, 5;}</style>
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<style data-pub-theme-postfix></style>
</head>
<body class="">
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=G-HR35NWP3CE" height="0"
width="0" style="display:none;visibility:hidden"></iframe></noscript>
<div id="entry">
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:71px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="flex-basis:0px;flex-grow:1;" class="logoContainer-p12gJb"><a href="/" native class="pencraft pc-display-contents pc-reset"><div draggable="false" class="pencraft pc-display-flex pc-position-relative pc-reset"><div style="width:40px;height:40px;" class="pencraft pc-display-flex pc-reset bg-white-ZBV5av pc-borderRadius-sm overflow-hidden-WdpwT6 sizing-border-box-DggLA4"><picture class="pencraft pc-display-contents pc-reset"><source type="image/webp" sizes="40px" srcset="https://substackcdn.com/image/fetch/$s_!NNuR!,w_40,h_40,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 40w, https://substackcdn.com/image/fetch/$s_!NNuR!,w_80,h_80,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 80w, https://substackcdn.com/image/fetch/$s_!NNuR!,w_120,h_120,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 120w" style="display:none;"/><img style="object-fit:cover;width:40px;height:40px;" src="https://substackcdn.com/image/fetch/$s_!NNuR!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png" srcset="https://substackcdn.com/image/fetch/$s_!NNuR!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 40w, https://substackcdn.com/image/fetch/$s_!NNuR!,w_80,h_80,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 80w, https://substackcdn.com/image/fetch/$s_!NNuR!,w_120,h_120,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 120w" sizes="40px" alt="Software at Scale" draggable="false" class="pencraft pc-reset"/></picture></div></div></a></div><div style="flex-grow:0;" class="titleContainer-DJYq5v"><h1 class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd"><a href="/" class="pencraft pc-display-contents pc-reset">Software at Scale</a></h1></div><div style="flex-basis:0px;flex-grow:1;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Fwhy-you-should-frequently-turn-down&amp;for_pub=reliability" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="height:72px;"></div></div></div><div><script nonce="oWvaPKxPuOo-NUgc7woNlQ" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down","mainEntityOfPage":"https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down","headline":"Why you should frequently turn down ~30% of canary instances","description":"The most effective (if scary) way to understand how your stateless service operates under load","image":[],"datePublished":"2020-11-03T16:51:18+00:00","dateModified":"2020-11-03T16:51:18+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"Utsav Shah","url":"https://substack.com/@reliability","description":"Software Engineer interested in building simple, self correcting and reliable systems. Twitter: @utsav_sha","identifier":"user:6567678","sameAs":["https://twitter.com/utsav_sha"],"image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!Ta5D!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!Ta5D!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg"}}],"publisher":{"@type":"Organization","name":"Software at Scale","url":"https://www.softwareatscale.dev","description":"Discussions on running software systems, teams, and products.","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":1000},"identifier":"pub:55725","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!NNuR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!NNuR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!NNuR!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!NNuR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!NNuR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!NNuR!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png"},"sameAs":["https://twitter.com/utsav_sha"]},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":4},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":0},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":0}]}</script><script nonce="oWvaPKxPuOo-NUgc7woNlQ" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Software at Scale","item":"https://www.softwareatscale.dev"},{"@type":"ListItem","position":2,"name":"Why you should frequently turn down ~30% of canary instances","item":"https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><h1 dir="auto" class="post-title published title-X77sOw">Why you should frequently turn down ~30% of canary instances</h1><h3 dir="auto" class="subtitle subtitle-HEEcLo">The most effective (if scary) way to understand how your stateless service operates under load</h3><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2020-11-03T16:51:18.309Z" title="2020-11-03T16:51:18.309Z">Nov 03, 2020</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (4)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">4</div></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-24" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p>If you’re building a production service that handles a large number of requests per second with high availability, it’s wise to understand how the service behaves when it’s overloaded. For example, with Python web servers, the scaling limits are often due to high memory usage. CPU utilization is generally low, and overloads manifest as higher rate of OOMs. This awareness can be crucial during an emergency.</p><p>However, it’s often unclear how to understand scalability limits for a service. A common approach is to start with a synthetic load via a script, which are most useful when the underlying service provides a relatively simple API (like Get, Put) and will behave similarly under production traffic. But this approach is not suitable if your service serves a large number of endpoints, has non trivial business logic, or has a fair number of dependencies that can affect availability. For example, a load testing script might not trigger an expensive case or an RPC to a fragile dependency. It also adds toil to maintain custom load testing scripts, and load testing tends to become a one-off process rather than an easily repeatable one.</p><p>An alternate approach is the Utilization DRT (Disaster Recovery Test).</p><h2 class="header-anchor-post"><strong>Utilization DRT</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§utilization-drt" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/17321148/utilization-drt" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>The goals are:</p><ul><li><p>Estimate the amount of headroom your service can handle in practice</p></li><li><p>Determine anomalies in behavior of your service when it’s overloaded</p></li></ul><p>To reliably run a production service - it’s important to understand its headroom - how much percent of current peak traffic it can handle without falling over. </p><p>Service owners should estimate a back of the napkin desired headroom percent - based on the service’s availability SLA, cost budget, traffic patterns and auto-scaling efficiency. For me, it’s generally 30 - 40% for a service that has to handle three 9s or more. Then one can use the utilization DRT to estimate what its headroom is so that capacity (or minimum autoscaled capacity) can be provisioned correctly.</p><p>Additionally, services tend to display unnatural behavior when under heavy load - garbage collection pausing too often under load is a standard problem for JVM based servers. Understanding this and tuning servers under controlled conditions is important so that they don’t fall over on high stress days due to unforeseen issues.</p><h3 class="header-anchor-post">Implementation<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§implementation" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/17321148/implementation" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>The steps are:</p><ul><li><p>One time - Create a dedicated canary cluster that is X% (usually 10%) of your production cluster</p></li><li><p>One time - Configure a request proxy (Envoy / Amazon API Gateway) to send X% of traffic to the canary cluster</p></li><li><p>One time - Set up a canary dashboard that tracks requests per second (RPS), number of available processes serving requests, and availability percentage</p></li><li><p>Turn down instances in the canary cluster one-by-one until it’s clear that with one more instance going down, users will see an availability hit</p></li></ul><p>This is similar in appearance to a “Chaos Monkey” test - but with a clear goal in mind - to understand the behavior of your cluster in practice. There’s several advantages of performing these.</p><h3 class="header-anchor-post">Advantages<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§advantages" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/17321148/advantages" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><strong>It mirrors production traffic</strong></p><p>This is the key advantage - it allows one to test complicated web servers without hand-written load generation scripts.</p><p><strong>It’s relatively simple to set up</strong></p><p>A request proxy and an isolated canary cluster is independently useful to have set up. After that - all you need a script that turns off web server processes, or more commonly, proxy sidecar processes, and a script to turn them back when it’s time to rollback the change. This also implies it’s easy to set up automation to do this process repeatedly.</p><p><strong>It’s easy to estimate overall cluster throughput</strong></p><p>Let’s say that a service’s peak load is 1,000 RPS, and it’s provisioned to a 100 pods, each running a single server process that can handle one request at a time. One can set aside 10 pods as the canary deployment, and turn them down until it seems like your request proxy has to retry a large number of requests to provide the appearance of reliability to clients. In this example - let’s say that that tipping point was 4 pods. We can then justifiably conclude that the overall cluster has 40% headroom for today’s load.</p><h3 class="header-anchor-post">Disadvantages<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§disadvantages" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/17321148/disadvantages" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>This approach has some unique downsides.</p><p><strong>It cannot easily be used for new services</strong></p><p>The recommendation for new services is to slowly ramp up traffic and add headroom as required.</p><p><strong>It doesn’t solve for overloaded dependencies</strong></p><p>One solution for learning how your service reacts when another service is acting overloaded is to run a DRT for that other service.</p><h2 class="header-anchor-post">FAQ<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§faq" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/17321148/faq" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><strong>Why not use auto-scaling so you never have to worry about capacity issues?</strong></p><p>Auto-scaling is a great solution to dynamically scale the number of instances - not only does it keep the system efficient, but also safe from client overload. However, auto-scaling is inherently reactive - a service will see an availability hit if there’s a large spike of requests that it cannot handle. So you will need to provision enough headroom to ensure that your service is not overloaded while auto-scaling kicks in, and then we’re back to square one - how much headroom is enough? Use a utilization DRT to confirm.</p><p><strong>Why not use serverless and not deal with these issues?</strong></p><p>Serverless/Lambdas tends to be at least 5x more expensive than running a server in AWS. At some point, the cost tradeoff is too hard to ignore, let alone other issues like cold start times.</p><h2 class="header-anchor-post">Conclusion<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§conclusion" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/17321148/conclusion" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Utilization DRTs are a useful tool for a reliability engineer to truly understand how their service will perform under overload and be more confident for scale ups or launches. Hopefully this post talks through a simple technique to run services more predictably and reliably.</p><p></p></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (4)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">4</div></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-53" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-left"><path d="m12 19-7-7 7-7"></path><path d="M19 12H5"></path></svg>Previous</button><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o">Next<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></button></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="visibility-check"></div><div id="discussion" class="pencraft pc-display-flex pc-flexDirection-column pc-gap-16 pc-paddingTop-32 pc-paddingBottom-32 pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-32 pc-reset container"><h4 class="pencraft pc-reset line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Discussion about this post</h4><div class="pencraft pc-alignSelf-flex-start pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Select discussion type" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-67-content-comments" data-state="active" id="radix-P0-67-trigger-comments" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Comments</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-67-content-restacks" data-state="inactive" id="radix-P0-67-trigger-restacks" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Restacks</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div></div></div><div id="substack-comments" class="single-post-section comments-section"><div class="container"><div class="visibility-check"></div><div data-test-id="comment-input" class="pencraft pc-display-flex pc-reset flex-grow-rzmknG"><form class="comment-input-form form-CkZ7Kt"><div style="--scale:32px;" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><div style="--scale:32px;" title="User" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-flexDirection-column pc-boxShadow-xs pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-sm"><textarea name="body" placeholder="Write a comment..." aria-label="Write a comment..." rows="4" class="pencraft input-qHk4bN autogrowing-_ipn9Y textarea-GbEjRX inputText-pV_yWb unstyled-IJLUJU"></textarea></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></form></div></div></div></div><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-80-content-top" data-state="active" id="radix-P0-80-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-80-content-new" data-state="inactive" id="radix-P0-80-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-80-content-community" data-state="inactive" id="radix-P0-80-trigger-community" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Discussions</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer themed-background"><div class="container"><div class="footer-terms"><span>© 2026 Utsav Shah</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&amp;utm_medium=web&amp;utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&amp;utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="oWvaPKxPuOo-NUgc7woNlQ">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
</div>
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
<script nonce="oWvaPKxPuOo-NUgc7woNlQ">
window.Sentry && window.Sentry.onLoad(function() {
window.Sentry.init({
environment: window._preloads.sentry_environment,
dsn: window._preloads.sentry_dsn,
})
})
</script>
<script nonce="oWvaPKxPuOo-NUgc7woNlQ">window._preloads = JSON.parse("{\"cspNonce\":\"oWvaPKxPuOo-NUgc7woNlQ\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://www.softwareatscale.dev\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":6567678,\"byline_images_enabled\":true,\"bylines_enabled\":false,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Utsav Shah\",\"cover_photo_url\":null,\"created_at\":\"2020-06-13T04:08:52.332Z\",\"custom_domain_optional\":false,\"custom_domain\":\"www.softwareatscale.dev\",\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":null,\"default_show_guest_bios\":true,\"email_banner_url\":null,\"email_from_name\":\"Software at Scale by Utsav\",\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":null,\"free_subscription_benefits\":null,\"ga_pixel_id\":\"G-HR35NWP3CE\",\"google_site_verification_token\":\"IORiP7_fax0ESrUYObrVUA0QF8qgq2BzPsISmpFzgfg\",\"google_tag_manager_token\":\"G-HR35NWP3CE\",\"hero_image\":null,\"hero_text\":\"Discussions on running software systems, teams, and products.\",\"hide_intro_subtitle\":false,\"hide_intro_title\":false,\"hide_podcast_feed_link\":false,\"homepage_type\":\"magaziney\",\"id\":55725,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":null,\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!NNuR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png\",\"minimum_group_size\":2,\"moderation_enabled\":true,\"name\":\"Software at Scale\",\"paid_subscription_benefits\":null,\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"disabled\",\"paywall_free_trial_enabled\":false,\"podcast_art_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/3b763612-d8e3-4199-95d2-d9e9a8d2a180_3000x3000.png\",\"paid_podcast_episode_art_url\":null,\"podcast_byline\":null,\"podcast_description\":\"Software at Scale is where we discuss the technical stories behind large software applications.\",\"podcast_enabled\":true,\"podcast_feed_url\":null,\"podcast_title\":null,\"post_preview_limit\":null,\"primary_user_id\":6567678,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"reliability\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#0068ef\",\"theme_var_color_links\":false,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":\"o6rzq\",\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":null,\"stripe_user_id\":null,\"stripe_country\":null,\"stripe_publishable_key\":null,\"stripe_platform_account\":null,\"automatic_tax_enabled\":null,\"author_name\":\"Utsav Shah\",\"author_handle\":\"reliability\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!Ta5D!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg\",\"author_bio\":\"Software Engineer interested in building simple, self correcting and reliable systems. Twitter: @utsav_sha\",\"twitter_screen_name\":\"utsav_sha\",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":\"#1a2e05\",\"web_bg_color\":\"#fafafa\",\"cover_bg_color\":\"#fafafa\",\"publication_id\":55725,\"color_links\":null,\"font_preset_heading\":\"heavy_sans\",\"font_preset_body\":\"sans\",\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"magaziney\",\"home_posts\":\"list\",\"home_show_top_posts\":false,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":null,\"custom_footer\":null,\"social_media_links\":null,\"font_options\":null,\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":{\"first_thread_email_sent_at\":\"2022-11-04T02:53:50.304+00:00\",\"activated_at\":\"2022-11-04T02:53:19.96+00:00\",\"reader_thread_notifications_enabled\":false,\"create_thread_minimum_role\":\"contributor\",\"photo_replies_enabled\":true,\"boost_free_subscriber_chat_preview_enabled\":false,\"push_suppression_enabled\":false},\"default_group_coupon_percent_off\":null,\"default_group_coupon_include_founding\":null,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2020-11-03T16:51:18.309Z\",\"has_podcast\":true,\"has_free_podcast\":true,\"has_subscriber_only_podcast\":false,\"has_community_content\":true,\"rankingDetail\":\"Launched 6 years ago\",\"rankingDetailFreeIncluded\":\"Thousands of subscribers\",\"rankingDetailOrderOfMagnitude\":0,\"rankingDetailFreeIncludedOrderOfMagnitude\":1000,\"rankingDetailFreeSubscriberCount\":\"Over 1,000 subscribers\",\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u062A\u0645 \u0627\u0644\u0625\u0637\u0644\u0627\u0642 6 years ago\"},\"ca\":{\"rankingDetail\":\"S\u2019ha llan\u00E7at fa 6 anys\"},\"da\":{\"rankingDetail\":\"Lancering 6 \u00E5r\"},\"de\":{\"rankingDetail\":\"vor 6 Jahren gelauncht\"},\"es\":{\"rankingDetail\":\"Lanzado hace 6 a\u00F1os\"},\"fr\":{\"rankingDetail\":\"Lanc\u00E9 il y a 6 ann\u00E9es\"},\"id\":{\"rankingDetail\":\"Diluncurkan 6 tahun yang lalu\"},\"ja\":{\"rankingDetail\":\"\u958B\u59CB\u65E5 6\u5E74\u524D\"},\"ko\":{\"rankingDetail\":\"6\uB144 \uC804\uC5D0 \uCD9C\uC2DC\"},\"nb\":{\"rankingDetail\":\"Lansert 6 \u00E5r\"},\"nl\":{\"rankingDetail\":\"Gelanceerd 6 jaar geleden\"},\"pl\":{\"rankingDetail\":\"Uruchomiono 6 lat temu\"},\"pt\":{\"rankingDetail\":\"Lan\u00E7ado 6 anos\"},\"pt-br\":{\"rankingDetail\":\"Lan\u00E7ado 6 anos\"},\"en-gb\":{\"rankingDetail\":\"Launched 6 years ago\"},\"it\":{\"rankingDetail\":\"Lanciato 6 anni\"},\"tr\":{\"rankingDetail\":\"6 y\u0131l ba\u015Flat\u0131ld\u0131\"},\"sv\":{\"rankingDetail\":\"Lanserad 6 \u00E5r sedan\"},\"fi\":{\"rankingDetail\":\"Launched 6 vuotta\"},\"is\":{\"rankingDetail\":\"Launched 6 \u00E1r\"},\"zh-hans\":{\"rankingDetail\":\"\u4E8E6 \u5E74\u524D\u4E0A\u7EBF\"},\"zh-hant\":{\"rankingDetail\":\"\u65BC 6 \u5E74\u524D \u63A8\u51FA\"},\"en\":{\"rankingDetail\":\"Launched 6 years ago\"}},\"freeSubscriberCount\":\"1,000\",\"freeSubscriberCountOrderOfMagnitude\":\"1.8K+\",\"author_bestseller_tier\":0,\"author_badge\":null,\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":true,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":\"2023-02-01T17:18:04.383Z\",\"primary_profile_name\":\"Utsav Shah\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!Ta5D!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg\",\"no_follow\":false,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"free\",\"sections\":[],\"podcastTabInfo\":null,\"multipub_migration\":null,\"navigationBarItems\":[],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Utsav Shah\",\"handle\":\"reliability\",\"role\":\"admin\",\"owner\":true,\"user_id\":6567678,\"photo_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg\",\"bio\":\"Software Engineer interested in building simple, self correcting and reliable systems. Twitter: @utsav_sha\",\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"threads_v2_enabled\":true,\"viralGiftsConfig\":null,\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://www.softwareatscale.dev\",\"hostname\":\"www.softwareatscale.dev\",\"is_on_substack\":false,\"spotify_podcast_settings\":{\"id\":4734,\"publication_id\":55725,\"section_id\":null,\"spotify_uri\":null,\"spotify_podcast_title\":null,\"created_at\":\"2024-04-11T18:05:40.194Z\",\"updated_at\":\"2024-04-11T18:05:40.194Z\",\"currently_published_on_spotify\":false,\"spotify_show_url\":\"\"},\"unified_podcast_settings\":null,\"podcastPalette\":{\"Vibrant\":{\"rgb\":[92,70,172],\"population\":94},\"DarkVibrant\":{\"rgb\":[49,39,85],\"population\":306},\"LightVibrant\":{\"rgb\":[145,196,232],\"population\":63},\"Muted\":{\"rgb\":[84,84,164],\"population\":2},\"DarkMuted\":{\"rgb\":[76,76,78],\"population\":46},\"LightMuted\":{\"rgb\":[138,165,191],\"population\":42}},\"pageThemes\":{\"podcast\":null},\"multiple_pins\":true,\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#1a2e05\",\"primary\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"primary_hover\":{\"r\":18,\"g\":30,\"b\":0,\"a\":1},\"primary_elevated\":{\"r\":18,\"g\":30,\"b\":0,\"a\":1},\"secondary\":{\"r\":26,\"g\":46,\"b\":5,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":26,\"g\":46,\"b\":5,\"a\":0.2},\"bg_hover\":{\"r\":26,\"g\":46,\"b\":5,\"a\":0.3},\"dark\":{\"primary\":{\"r\":59,\"g\":80,\"b\":39,\"a\":1},\"primary_hover\":{\"r\":76,\"g\":97,\"b\":55,\"a\":1},\"primary_elevated\":{\"r\":76,\"g\":97,\"b\":55,\"a\":1},\"secondary\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.2},\"bg_hover\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":112,\"g\":135,\"b\":90,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":206.90096477355226,\"g\":206.90096477355175,\"b\":206.9009647735519,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#1a2e05\",\"primary\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"primary_hover\":{\"r\":13,\"g\":23,\"b\":3,\"a\":1},\"primary_elevated\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"secondary\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":59,\"g\":80,\"b\":39,\"a\":1},\"primary_hover\":{\"r\":76,\"g\":97,\"b\":55,\"a\":1},\"primary_elevated\":{\"r\":76,\"g\":97,\"b\":55,\"a\":1},\"secondary\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.2},\"bg_hover\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"secondary\":{\"r\":132,\"g\":133,\"b\":133,\"a\":1},\"tertiary\":{\"r\":144,\"g\":144,\"b\":144,\"a\":1},\"accent\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":112,\"g\":135,\"b\":90,\"a\":1}}},\"bg\":{\"name\":\"#fafafa\",\"hue\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"tint\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_hover\":{\"r\":235,\"g\":235,\"b\":235,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":235,\"g\":235,\"b\":235,\"a\":1},\"secondary_elevated\":{\"r\":235,\"g\":235,\"b\":235,\"a\":1},\"tertiary\":{\"r\":216,\"g\":216,\"b\":216,\"a\":1},\"quaternary\":{\"r\":179,\"g\":179,\"b\":179,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}},\"wordmark_bg\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"is_dark\":false},\"fonts\":{\"heading\":\"heavy_sans\",\"body\":\"sans\"}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[92,70,172],\"population\":94},\"DarkVibrant\":{\"rgb\":[49,40,85],\"population\":306},\"LightVibrant\":{\"rgb\":[148,204,236],\"population\":1},\"Muted\":{\"rgb\":[84,84,164],\"population\":2},\"DarkMuted\":{\"rgb\":[74,83,114],\"population\":24},\"LightMuted\":{\"rgb\":[179,184,190],\"population\":343}}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":false,\"customDomain\":\"www.softwareatscale.dev\"},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":false,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":false,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":\"G-HR35NWP3CE\",\"twitterPixelId\":\"o6rzq\",\"parselyPixelId\":null,\"googleTagManagerToken\":\"G-HR35NWP3CE\",\"googleAnalytics4Token\":null,\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":false,\"credit_token_enabled\":false,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":true,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":true,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":null,\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"\",\"notes_feed_enabled\":true,\"paywall_unlock_tokens\":false,\"post_preview_crop_gravity\":\"center\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":false,\"reader_referrals_leaderboard_enabled\":false,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":true,\"local_currency_modal_seen\":false,\"local_payment_methods_modal_seen\":false,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":false,\"welcome_page_opt_out_text\":\"No thanks\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":false,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":17321148,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2020-11-03T16:51:18.309Z\",\"updated_at\":\"2023-01-26T13:45:13.356Z\",\"publication_id\":55725,\"search_engine_description\":null,\"search_engine_title\":null,\"section_id\":null,\"should_send_free_preview\":false,\"show_guest_bios\":false,\"slug\":\"why-you-should-frequently-turn-down\",\"social_title\":null,\"subtitle\":\"The most effective (if scary) way to understand how your stateless service operates under load\",\"teaser_post_eligible\":true,\"title\":\"Why you should frequently turn down ~30% of canary instances\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"everyone\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":0,\"reactions\":{\"\u2764\":4},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"coming-soon\",\"next_post_slug\":\"go-for-internal-services\",\"cover_image\":null,\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":null,\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"The most effective (if scary) way to understand how your stateless service operates under load\",\"body_html\":\"<p>If you\u2019re building a production service that handles a large number of requests per second with high availability, it\u2019s wise to understand how the service behaves when it\u2019s overloaded. For example, with Python web servers, the scaling limits are often due to high memory usage. CPU utilization is generally low, and overloads manifest as higher rate of OOMs. This awareness can be crucial during an emergency.</p><p>However, it\u2019s often unclear how to understand scalability limits for a service. A common approach is to start with a synthetic load via a script, which are most useful when the underlying service provides a relatively simple API (like Get, Put) and will behave similarly under production traffic. But this approach is not suitable if your service serves a large number of endpoints, has non trivial business logic, or has a fair number of dependencies that can affect availability. For example, a load testing script might not trigger an expensive case or an RPC to a fragile dependency. It also adds toil to maintain custom load testing scripts, and load testing tends to become a one-off process rather than an easily repeatable one.</p><p>An alternate approach is the Utilization DRT (Disaster Recovery Test).</p><h2><strong>Utilization DRT</strong></h2><p>The goals are:</p><ul><li><p>Estimate the amount of headroom your service can handle in practice</p></li><li><p>Determine anomalies in behavior of your service when it\u2019s overloaded</p></li></ul><p>To reliably run a production service - it\u2019s important to understand its headroom - how much percent of current peak traffic it can handle without falling over. </p><p>Service owners should estimate a back of the napkin desired headroom percent - based on the service\u2019s availability SLA, cost budget, traffic patterns and auto-scaling efficiency. For me, it\u2019s generally 30 - 40% for a service that has to handle three 9s or more. Then one can use the utilization DRT to estimate what its headroom is so that capacity (or minimum autoscaled capacity) can be provisioned correctly.</p><p>Additionally, services tend to display unnatural behavior when under heavy load - garbage collection pausing too often under load is a standard problem for JVM based servers. Understanding this and tuning servers under controlled conditions is important so that they don\u2019t fall over on high stress days due to unforeseen issues.</p><h3>Implementation</h3><p>The steps are:</p><ul><li><p>One time - Create a dedicated canary cluster that is X% (usually 10%) of your production cluster</p></li><li><p>One time - Configure a request proxy (Envoy / Amazon API Gateway) to send X% of traffic to the canary cluster</p></li><li><p>One time - Set up a canary dashboard that tracks requests per second (RPS), number of available processes serving requests, and availability percentage</p></li><li><p>Turn down instances in the canary cluster one-by-one until it\u2019s clear that with one more instance going down, users will see an availability hit</p></li></ul><p>This is similar in appearance to a \u201CChaos Monkey\u201D test - but with a clear goal in mind - to understand the behavior of your cluster in practice. There\u2019s several advantages of performing these.</p><h3>Advantages</h3><p><strong>It mirrors production traffic</strong></p><p>This is the key advantage - it allows one to test complicated web servers without hand-written load generation scripts.</p><p><strong>It\u2019s relatively simple to set up</strong></p><p>A request proxy and an isolated canary cluster is independently useful to have set up. After that - all you need a script that turns off web server processes, or more commonly, proxy sidecar processes, and a script to turn them back when it\u2019s time to rollback the change. This also implies it\u2019s easy to set up automation to do this process repeatedly.</p><p><strong>It\u2019s easy to estimate overall cluster throughput</strong></p><p>Let\u2019s say that a service\u2019s peak load is 1,000 RPS, and it\u2019s provisioned to a 100 pods, each running a single server process that can handle one request at a time. One can set aside 10 pods as the canary deployment, and turn them down until it seems like your request proxy has to retry a large number of requests to provide the appearance of reliability to clients. In this example - let\u2019s say that that tipping point was 4 pods. We can then justifiably conclude that the overall cluster has 40% headroom for today\u2019s load.</p><h3>Disadvantages</h3><p>This approach has some unique downsides.</p><p><strong>It cannot easily be used for new services</strong></p><p>The recommendation for new services is to slowly ramp up traffic and add headroom as required.</p><p><strong>It doesn\u2019t solve for overloaded dependencies</strong></p><p>One solution for learning how your service reacts when another service is acting overloaded is to run a DRT for that other service.</p><h2>FAQ</h2><p><strong>Why not use auto-scaling so you never have to worry about capacity issues?</strong></p><p>Auto-scaling is a great solution to dynamically scale the number of instances - not only does it keep the system efficient, but also safe from client overload. However, auto-scaling is inherently reactive - a service will see an availability hit if there\u2019s a large spike of requests that it cannot handle. So you will need to provision enough headroom to ensure that your service is not overloaded while auto-scaling kicks in, and then we\u2019re back to square one - how much headroom is enough? Use a utilization DRT to confirm.</p><p><strong>Why not use serverless and not deal with these issues?</strong></p><p>Serverless/Lambdas tends to be at least 5x more expensive than running a server in AWS. At some point, the cost tradeoff is too hard to ignore, let alone other issues like cold start times.</p><h2>Conclusion</h2><p>Utilization DRTs are a useful tool for a reliability engineer to truly understand how their service will perform under overload and be more confident for scale ups or launches. Hopefully this post talks through a simple technique to run services more predictably and reliably.</p><p></p>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"If you\u2019re building a production service that handles a large number of requests per second with high availability, it\u2019s wise to understand how the service behaves when it\u2019s overloaded. For example, with Python web servers, the scaling limits are often due to high memory usage. CPU utilization is generally low, and overloads manifest as higher rate of OO\u2026\",\"wordcount\":936,\"post_preview_limit\":null,\"language\":\"en\",\"postTags\":[],\"postCountryBlocks\":[],\"headlineTest\":null,\"publishedBylines\":[{\"id\":6567678,\"name\":\"Utsav Shah\",\"handle\":\"reliability\",\"photo_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg\",\"bio\":\"Software Engineer interested in building simple, self correcting and reliable systems. Twitter: @utsav_sha\",\"profile_set_up_at\":\"2022-02-28T22:12:41.936Z\",\"reader_installed_at\":\"2022-03-09T16:36:54.472Z\",\"publicationUsers\":[{\"id\":251742,\"user_id\":6567678,\"publication_id\":55725,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":55725,\"name\":\"Software at Scale\",\"subdomain\":\"reliability\",\"custom_domain\":\"www.softwareatscale.dev\",\"custom_domain_optional\":false,\"hero_text\":\"Discussions on running software systems, teams, and products.\",\"logo_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png\",\"author_id\":6567678,\"primary_user_id\":6567678,\"theme_var_background_pop\":\"#0068ef\",\"created_at\":\"2020-06-13T04:08:52.332Z\",\"email_from_name\":\"Software at Scale by Utsav\",\"copyright\":\"Utsav Shah\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"magaziney\",\"is_personal_mode\":false,\"logo_url_wide\":null}},{\"id\":6993066,\"user_id\":6567678,\"publication_id\":6852092,\"role\":\"admin\",\"public\":true,\"is_primary\":false,\"publication\":{\"id\":6852092,\"name\":\"Questions Worth Asking\",\"subdomain\":\"nishkama2\",\"custom_domain\":null,\"custom_domain_optional\":false,\"hero_text\":\"\",\"logo_url\":null,\"author_id\":6567678,\"primary_user_id\":null,\"theme_var_background_pop\":\"#FF6719\",\"created_at\":\"2025-11-09T14:04:08.104Z\",\"email_from_name\":\"Utsav from Questions Worth Asking\",\"copyright\":\"Utsav Shah\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"newspaper\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"twitter_screen_name\":\"utsav_sha\",\"is_guest\":false,\"bestseller_tier\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":4,\"comment_count\":0,\"child_comment_count\":0,\"is_geoblocked\":false,\"hasCashtag\":false},\"comments\":[],\"canonicalUrl\":\"https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":\"#1a2e05\",\"background_pop\":\"#1a2e05\",\"color_theme_bg_web\":\"#fafafa\",\"cover_bg_color\":\"#fafafa\",\"cover_bg_color_secondary\":\"#ebebeb\",\"background_pop_darken\":\"#0d1703\",\"print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_darken\":\"#0d1703\",\"color_theme_print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_20\":\"rgba(26, 46, 5, 0.2)\",\"color_theme_bg_pop_30\":\"rgba(26, 46, 5, 0.3)\",\"print_pop\":\"#1a2e05\",\"color_theme_accent\":\"#1a2e05\",\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#1a2e05\",\"font_family_headings_preset\":\"'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'\",\"font_weight_headings_preset\":900,\"font_family_body_preset\":\"'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'\",\"font_weight_body_preset\":400,\"font_preset_heading\":\"heavy_sans\",\"font_preset_body\":\"sans\",\"home_hero\":\"magaziney\",\"home_posts\":\"list\",\"web_bg_color\":\"#fafafa\",\"background_contrast_1\":\"#ebebeb\",\"background_contrast_2\":\"#d8d8d8\",\"background_contrast_3\":\"#b3b3b3\",\"background_contrast_4\":\"#909090\",\"background_contrast_5\":\"#4f4f4f\",\"color_theme_bg_contrast_1\":\"#ebebeb\",\"color_theme_bg_contrast_2\":\"#d8d8d8\",\"color_theme_bg_contrast_3\":\"#b3b3b3\",\"color_theme_bg_contrast_4\":\"#909090\",\"color_theme_bg_contrast_5\":\"#4f4f4f\",\"color_theme_bg_elevated\":\"#fafafa\",\"color_theme_bg_elevated_secondary\":\"#ebebeb\",\"color_theme_bg_elevated_tertiary\":\"#d8d8d8\",\"color_theme_detail\":\"#e1e1e1\",\"background_contrast_pop\":\"rgba(26, 46, 5, 0.4)\",\"color_theme_bg_contrast_pop\":\"rgba(26, 46, 5, 0.4)\",\"theme_bg_is_dark\":\"0\",\"print_on_web_bg_color\":\"#363737\",\"print_secondary_on_web_bg_color\":\"#848585\",\"background_pop_rgb\":\"26, 46, 5\",\"color_theme_bg_pop_rgb\":\"26, 46, 5\",\"color_theme_accent_rgb\":\"26, 46, 5\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":null,\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"125.0\",\"major\":\"125\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
<script nonce="oWvaPKxPuOo-NUgc7woNlQ">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"reliability\",\"publication_id\":55725,\"has_plans\":false,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.cd6bda92.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.e64cd9e7.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.70d2064a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.2aec390a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.85be5638.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.6ae0efc1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.dc76f9b7.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.f236be4c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.8420ae5a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.d8bb2330.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.3f1d4841.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.13cbe240.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.4e2d2604.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.e3915e76.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.7bc45bbd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.5032c27a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.49d00a24.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.39bfbf77.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.240bb9de.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.d381c46e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.bceee863.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.25ec383f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.61e2d663.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.bc160fda.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.df2700cc.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.9bd8805f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89356.d1e9cc24.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.d77360e7.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47925.8f59f90c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.6b19c4da.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71505.926eeed0.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.51367581.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.1f9cab9a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.babfb9f2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/86118.3ef602a8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.b709da20.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.8204204e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.482f05bb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/29231.dff155cd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.ecb6663c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.2c454bc2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.10e3f4a5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.24f948b9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.bee38989.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.b71dbf43.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.ec777231.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.b1934b97.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.a8bf9e5a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.5b1bb3ae.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.44f8ea11.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.a75fa23f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.a779610d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.205ac27b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.9eb856ae.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.22720df6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.9d23be71.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58483.e9de57a4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.068b9ac2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.962d3acc.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.f1c7d8d1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/92610.794e210b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.0032d73d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11244.d96d70e6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99260.b43290e5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.13a56340.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/76768.3eb4f626.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/41932.d646d645.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12114.bf01e811.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.91843f70.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.36fd19e4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53871.e30a7e0d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.8898ca60.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99068.1e3c1d90.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.914c5b02.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.b768c624.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.e1599a8f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.9c81506d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.2cdde67c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.cc7e8a2e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.3a4b3814.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/32552.0dc9b45b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.c4446c28.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.29f36660.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.0de8143e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.89ea1e18.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.a1f497b9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.d802b20e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.dce45db9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.4a89b9ee.js" charset="utf-8"></script>
<script nonce="oWvaPKxPuOo-NUgc7woNlQ" nomodule>
(function() {
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
var warningDiv = document.createElement('div');
warningDiv.style.color = 'red';
warningDiv.style.padding = '10px';
warningDiv.style.margin = '10px 0';
warningDiv.style.border = '1px solid red';
warningDiv.style.backgroundColor = 'lightyellow';
warningDiv.innerText = message;
document.body.prepend(warningDiv);
})();
</script>
<!-- Datadog Analytics -->
<script nonce="oWvaPKxPuOo-NUgc7woNlQ">
(function(h,o,u,n,d) {
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
d=o.createElement(u);d.async=1;d.src=n
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
window.DD_RUM.onReady(function() {
window.DD_RUM.init({
clientToken: 'puba71073f072643721169b68f352438710',
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
site: 'datadoghq.com',
service: 'web',
env: window._preloads.dd_env,
version: '8f93c6cc22acb7484a25b4a2d3cb21e443220162',
sessionSampleRate: 1,
sessionReplaySampleRate: 10,
trackUserInteractions: window._preloads.dd_ti,
trackResources: true,
trackLongTasks: true,
defaultPrivacyLevel: 'mask-user-input',
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
});
})
</script>
<!-- End Datadog Analytics -->
<!-- Cloudflare Web Analytics -->
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
<!-- End Cloudflare Web Analytics -->
<!-- Fallback tracking pixels -->
<noscript>
<img height="1" width="1" style="display:none" alt="" src="https://analytics.twitter.com/i/adsct?p_id=Twitter&p_user_id=0&txn_id=o6rzq&events=%5B%5B%22pageview%22%2Cnull%5D%5D&tw_sale_amount=0&tw_order_quantity=0&tw_iframe_status=0"/>
<img height="1" width="1" style="display:none" alt="" src="https://t.co/i/adsct?p_id=Twitter&p_user_id=0&txn_id=o6rzq&events=%5B%5B%22pageview%22%2Cnull%5D%5D&tw_sale_amount=0&tw_order_quantity=0&tw_iframe_status=0"/>
</noscript>
<script nonce="oWvaPKxPuOo-NUgc7woNlQ">
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='https://static.ads-twitter.com/uwt.js',
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
twq('config','o6rzq');
</script>
<noscript>
<style>
#nojs-banner {
position: fixed;
bottom: 0;
left: 0;
padding: 16px 16px 16px 32px;
width: 100%;
box-sizing: border-box;
background: red;
color: white;
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
font-size: 13px;
line-height: 13px;
}
#nojs-banner a {
color: inherit;
text-decoration: underline;
}
</style>
<div id="nojs-banner">
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
</div>
</noscript>
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="oWvaPKxPuOo-NUgc7woNlQ" data-cf-beacon='{"version":"2024.11.0","token":"68cfe66b5c4749e2ba64d4d9640c04c0","spa":2}' crossorigin="anonymous"></script>
</body>
</html>