495 lines
207 KiB
HTML
495 lines
207 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en" dir="ltr">
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
|
||
|
||
|
||
<link rel="preconnect" href="https://substackcdn.com" />
|
||
|
||
|
||
|
||
<title data-rh="true">Metastable Failures Explained: Why Fixing the Trigger Fails</title>
|
||
|
||
<meta data-rh="true" name="theme-color" content="#f8ede3"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Metastable Failures Explained: Why Fixing the Trigger Fails"/><meta data-rh="true" name="twitter:title" content="Metastable Failures Explained: Why Fixing the Trigger Fails"/><meta data-rh="true" name="description" content="Why some outages keep going long after the trigger is gone. A guide to metastable failures, feedback loops, and hidden capacity in distributed systems."/><meta data-rh="true" property="og:description" content="The system is technically up, effectively down."/><meta data-rh="true" name="twitter:description" content="The system is technically up, effectively down."/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!Su99!,w_1200,h_675,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!vQpX!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fthecodercafe.substack.com%2Fapi%2Fv1%2Fpost_preview%2F194114637%2Ftwitter.jpg%3Fversion%3D4"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" property="article:modified_time" content="2026-05-27T13:07:23.467Z"/>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@layer legacy, pencraftReset, pencraft;
|
||
</style>
|
||
|
||
|
||
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
|
||
|
||
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/color_links.9856f2f08353fdd05959.css" />
|
||
|
||
|
||
|
||
<link rel="preload" as="font" href="https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M5knjsS_ul.woff2" crossorigin />
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.cee9fbee.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/86118.c25b9f67.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/76768.f15fc605.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/41932.97d2663c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/32552.313415d5.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
|
||
<meta name="author" content="Teiva Harsanyi" />
|
||
<meta property="og:url" content="https://read.thecoder.cafe/p/metastable-failures" />
|
||
|
||
|
||
<link rel="canonical" href="https://read.thecoder.cafe/p/metastable-failures" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="shortcut icon" href="https://substackcdn.com/image/fetch/$s_!kMQx!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Ffavicon.ico">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="16x16" href="https://substackcdn.com/image/fetch/$s_!TONI!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Ffavicon-16x16.png">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="32x32" href="https://substackcdn.com/image/fetch/$s_!mK2L!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Ffavicon-32x32.png">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="48x48" href="https://substackcdn.com/image/fetch/$s_!RuxL!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Ffavicon-48x48.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="57x57" href="https://substackcdn.com/image/fetch/$s_!uB8S!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-57x57.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="60x60" href="https://substackcdn.com/image/fetch/$s_!4efr!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-60x60.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="72x72" href="https://substackcdn.com/image/fetch/$s_!FFbp!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-72x72.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="76x76" href="https://substackcdn.com/image/fetch/$s_!2I_z!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-76x76.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="114x114" href="https://substackcdn.com/image/fetch/$s_!PShy!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-114x114.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="120x120" href="https://substackcdn.com/image/fetch/$s_!1tUn!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-120x120.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="144x144" href="https://substackcdn.com/image/fetch/$s_!QMWd!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-144x144.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="152x152" href="https://substackcdn.com/image/fetch/$s_!zdXb!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-152x152.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="167x167" href="https://substackcdn.com/image/fetch/$s_!gSXD!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-167x167.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="180x180" href="https://substackcdn.com/image/fetch/$s_!Ykmt!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-180x180.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="1024x1024" href="https://substackcdn.com/image/fetch/$s_!hQRn!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1db0a83a-2a01-4e3b-a101-570d7352394e%2Fapple-touch-icon-1024x1024.png">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="alternate" type="application/rss+xml" href="/feed?sectionId=206570" title="The Coder Cafe"/>
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8on7mTNmnUHowCw.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onXmTNmnUHowCw.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onTmTNmnUHowCw.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onrmTNmnUHo.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M9knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M2knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M3knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M5knjsS_ul.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3FafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3OafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3PafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3BafaPWnII.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||
|
||
</style>
|
||
|
||
|
||
|
||
|
||
|
||
<style data-pub-theme-prefix>:root{--color_theme_bg_pop:#8d493a;--background_pop:#8d493a;--color_theme_bg_web:#f8ede3;--cover_bg_color:#f8ede3;--cover_bg_color_secondary:#e9dfd5;--background_pop_darken:#7b4033;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#7b4033;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(141, 73, 58, 0.2);--color_theme_bg_pop_30:rgba(141, 73, 58, 0.3);--print_pop:#8d493a;--color_theme_accent:#8d493a;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#8d493a;--home_hero:newspaper;--home_posts:custom;--home_show_top_posts:true;--web_bg_color:#f8ede3;--background_contrast_1:#e9dfd5;--background_contrast_2:#d6cdc4;--background_contrast_3:#b2aaa3;--background_contrast_4:#8e8882;--background_contrast_5:#4e4b48;--color_theme_bg_contrast_1:#e9dfd5;--color_theme_bg_contrast_2:#d6cdc4;--color_theme_bg_contrast_3:#b2aaa3;--color_theme_bg_contrast_4:#8e8882;--color_theme_bg_contrast_5:#4e4b48;--color_theme_bg_elevated:#f8ede3;--color_theme_bg_elevated_secondary:#e9dfd5;--color_theme_bg_elevated_tertiary:#d6cdc4;--color_theme_detail:#dfd5cc;--background_contrast_pop:rgba(141, 73, 58, 0.4);--color_theme_bg_contrast_pop:rgba(141, 73, 58, 0.4);--theme_bg_is_dark:0;--print_on_web_bg_color:hsl(28.571428571428594, 15.688073394495422%, 28.54901960784314%);--print_secondary_on_web_bg_color:#84807c;--background_pop_rgb:141, 73, 58;--color_theme_bg_pop_rgb:141, 73, 58;--color_theme_accent_rgb:141, 73, 58;}</style>
|
||
|
||
|
||
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
|
||
|
||
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/color_links.9856f2f08353fdd05959.css" />
|
||
|
||
|
||
<style data-pub-theme-postfix></style>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script async="true" src="https://www.googletagmanager.com/gtag/js?id=G-M9NDKF5ZJX&l=GA4DataLayer" >
|
||
</script>
|
||
|
||
</head>
|
||
|
||
<body class="">
|
||
|
||
|
||
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KDC62RGC" height="0"
|
||
width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
|
||
|
||
|
||
<div id="entry">
|
||
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:87px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-paddingTop-0 pc-paddingBottom-0 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="visibility:visible;flex:1 0 0;" class="logoContainer-p12gJb"><a href="/" native class="pencraft pc-display-contents pc-reset"><div draggable="false" class="pencraft pc-display-flex pc-position-relative pc-reset"><div style="width:40px;height:40px;" class="pencraft pc-display-flex pc-reset bg-white-ZBV5av pc-borderRadius-sm overflow-hidden-WdpwT6 sizing-border-box-DggLA4"><picture class="pencraft pc-display-contents pc-reset"><source type="image/webp" sizes="40px" srcset="https://substackcdn.com/image/fetch/$s_!OZXv!,w_40,h_40,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png 40w, https://substackcdn.com/image/fetch/$s_!OZXv!,w_80,h_80,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png 80w, https://substackcdn.com/image/fetch/$s_!OZXv!,w_120,h_120,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png 120w" style="display:none;"/><img style="object-fit:cover;width:40px;height:40px;" src="https://substackcdn.com/image/fetch/$s_!OZXv!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png" srcset="https://substackcdn.com/image/fetch/$s_!OZXv!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png 40w, https://substackcdn.com/image/fetch/$s_!OZXv!,w_80,h_80,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png 80w, https://substackcdn.com/image/fetch/$s_!OZXv!,w_120,h_120,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png 120w" sizes="40px" alt="The Coder Cafe" draggable="false" class="pencraft pc-reset"/></picture></div></div></a></div><div style="flex-basis:0px;flex-grow:2;min-width:0px;display:flex;justify-content:center;" class="logoTopCenterSlot-hss6WZ"><div style="flex-grow:0;" class="titleContainer-DJYq5v overflowVisible-joA5t0"><h1 id="wordlogo" class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd titleWithWordmark-GfqxEZ"><a href="/" class="pencraft pc-display-contents pc-reset"><img alt="The Coder Cafe" src="https://substackcdn.com/image/fetch/$s_!xko8!,e_trim:10:white/e_trim:10:transparent/h_72,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09e5dcc0-f9fe-47f0-a81e-0148d03f7535_1344x256.png" style="max-height:36px;" class="wordmark-WcDTTu"/></a></h1></div></div><div style="flex-basis:0px;flex-grow:1;min-width:fit-content;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Fmetastable-failures&for_pub=thecodercafe" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="background-color:#f8ede3;height:88px;"></div></div></div><div><script nonce="SiMLZxbT83ypi3YnctfvQQ" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://read.thecoder.cafe/p/metastable-failures","mainEntityOfPage":"https://read.thecoder.cafe/p/metastable-failures","headline":"Metastable Failures Explained: Why Fixing the Trigger Fails","description":"Why some outages keep going long after the trigger is gone. A guide to metastable failures, feedback loops, and hidden capacity in distributed systems.","image":[{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!Su99!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png"}],"datePublished":"2026-05-27T13:02:50+00:00","dateModified":"2026-05-27T13:02:50+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"Teiva Harsanyi","url":"https://substack.com/@teivah","description":"Software Engineer at Google, Author, Speaker.","identifier":"user:3529974","image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!XdES!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!XdES!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png"}}],"publisher":{"@type":"Organization","name":"The Coder Cafe","url":"https://read.thecoder.cafe","description":"AI is getting better every day. Are you? At The Coder Cafe, we serve fundamental concepts to make you an engineer that AI won't replace. Written by a Google SWE, trusted by thousands of engineers worldwide.","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":1000},"identifier":"pub:2874913","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!OZXv!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!OZXv!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!OZXv!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!OZXv!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!OZXv!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!OZXv!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png"}},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":12},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":1},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":2}]}</script><script nonce="SiMLZxbT83ypi3YnctfvQQ" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"The Coder Cafe","item":"https://read.thecoder.cafe"},{"@type":"ListItem","position":2,"name":"Metastable Failures Explained: Why Fixing the Trigger Fails","item":"https://read.thecoder.cafe/p/metastable-failures"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><div class="pencraft pc-display-flex pc-reset post-label"><a href="https://read.thecoder.cafe/s/concepts/?utm_source=substack&utm_medium=menu" class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-14-MLPa7j weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA">Concepts</a></div><h1 dir="auto" class="post-title published title-X77sOw">Metastable Failures in Distributed Systems</h1><h3 dir="auto" class="subtitle subtitle-HEEcLo">Why Fixing the Trigger Fails</h3><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-row pc-gap-8 pc-alignItems-center pc-justifyContent-flex-start pc-reset"><div style="--scale:36px;--offset:9px;--border-width:4.5px;" class="pencraft pc-display-flex pc-flexDirection-row pc-alignItems-center pc-justifyContent-flex-start pc-reset ltr-qDBmby"><a href="https://substack.com/@teivah" aria-label="View Teiva Harsanyi's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:36px;" tabindex="0" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6 avatar-u8q6xB last-JfNEJ_"><div style="--scale:36px;" title="Teiva Harsanyi" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!XdES!,w_36,h_36,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png 1x, https://substackcdn.com/image/fetch/$s_!XdES!,w_72,h_72,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png 2x, https://substackcdn.com/image/fetch/$s_!XdES!,w_108,h_108,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!XdES!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png" alt="Teiva Harsanyi's avatar" srcset="https://substackcdn.com/image/fetch/$s_!XdES!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png 1x, https://substackcdn.com/image/fetch/$s_!XdES!,w_72,h_72,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png 2x, https://substackcdn.com/image/fetch/$s_!XdES!,w_108,h_108,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png 3x" width="36" height="36" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-reset color-primary-zABazT line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><span style="min-width:0;" data-state="closed"><a href="https://substack.com/@teivah" class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ">Teiva Harsanyi</a></span></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2026-05-27T13:02:50.954Z" title="2026-05-27T13:02:50.954Z">May 27, 2026</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (12)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">12</div></button></div><button tabindex="0" type="button" aria-label="View comments (2)" data-href="https://read.thecoder.cafe/p/metastable-failures/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg><div class="label">2</div></button><button tabindex="0" type="button" id="radix-P0-27" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">1</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p><span>☕ </span><em><span>Welcome to The Coder Cafe! Today, we explore one of the nastiest failure patterns in distributed systems: metastable failures. Based on the </span><a href="https://sigops.org/s/conferences/hotos/2021/papers/hotos21-s11-bronson.pdf">Metastable Failures in Distributed Systems</a><span> whitepaper, we break down why these failures happen, why they persist, what we can do about them, and why our instinct to fix them is probably wrong. Get cozy, grab a coffee, and let’s begin!</span></em></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!Su99!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!Su99!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 424w, https://substackcdn.com/image/fetch/$s_!Su99!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 848w, https://substackcdn.com/image/fetch/$s_!Su99!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 1272w, https://substackcdn.com/image/fetch/$s_!Su99!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!Su99!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png" width="1456" height="728" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/e51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":728,"width":1456,"resizeWidth":null,"bytes":1646402,"alt":"The system is technically up, effectively down.","title":null,"type":"image/png","href":null,"belowTheFold":false,"topImage":true,"internalRedirect":"https://read.thecoder.cafe/i/194114637?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png","isProcessing":false,"align":null,"offset":false}" alt="The system is technically up, effectively down." title="The system is technically up, effectively down." srcset="https://substackcdn.com/image/fetch/$s_!Su99!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 424w, https://substackcdn.com/image/fetch/$s_!Su99!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 848w, https://substackcdn.com/image/fetch/$s_!Su99!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 1272w, https://substackcdn.com/image/fetch/$s_!Su99!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 1456w" sizes="100vw" fetchpriority="high" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><h1 class="header-anchor-post">Stable, Vulnerable, Metastable<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§stable-vulnerable-metastable" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.thecoder.cafe/i/194114637/stable-vulnerable-metastable" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><p><span>Metastable failures borrow their name from physics, where </span><strong>metastable means something that looks stable but isn’t</strong><span>. To understand how a distributed system can end up in such a state, we need to look at three distinct states it can be in:</span></p><ul><li><p><strong>Stable:</strong><span> The system recovers on its own after any disruption. This is what we call resilience in </span><em><a href="https://read.thecoder.cafe/p/resilient-fault-tolerant-robust-reliable">Resilient, Fault-tolerant, Robust, or Reliable</a></em><span>.</span></p></li><li><p><strong>Vulnerable</strong><span>: The system looks perfectly healthy, but it's operating above its </span><strong>hidden capacity</strong><span>: the load level below which it can self-heal from any disruption. It responds fast, metrics are green, and nothing is alarming. Many production systems deliberately operate here because it's more efficient: resources are used closer to their limit. But there's </span><strong>no slack left</strong><span>. And the deeper the system operates in a vulnerable state, the smaller the trigger needed to push it over the edge. Indeed, a system just above its hidden capacity can survive large disruptions; a system near its advertised capacity can be tipped by almost anything.</span></p></li><li><p><strong>Metastable failure</strong><span>: A trigger (e.g., a network blip, a deployment, a traffic spike) pushes the system over its hidden capacity. The system is not fully broken: processes are alive, and it’s still running. But goodput collapses: it’s no longer doing any useful work. </span><strong>Technically up, effectively down</strong><span>. And unlike a regular outage, removing the trigger doesn’t fix it. Getting out requires a strong corrective push: a restart, a dramatic load reduction, a manual intervention.</span></p><blockquote><p><strong>NOTE</strong><span>: </span><em>If you’re not familiar with the concept of goodput, it’s the throughput of useful work completed successfully. For example, in a web application receiving 1000 requests per second but returning errors for 800 of them, the goodput is only 200 RPS.</em></p></blockquote></li></ul><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!qhkv!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!qhkv!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 424w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 848w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 1272w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!qhkv!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png" width="1456" height="320" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/a54abb43-fd09-4675-bb91-b695db389435_1640x360.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":320,"width":1456,"resizeWidth":null,"bytes":56976,"alt":"Diagram showing the three states of a system experiencing a metastable failure: Stable, Vulnerable, and Metastable. Arrows show transitions: increasing load moves the system from Stable to Vulnerable, a trigger moves it from Vulnerable to Metastable, and a strong corrective action (dashed arrow) brings it back to Stable.","title":null,"type":"image/png","href":null,"belowTheFold":false,"topImage":false,"internalRedirect":"https://read.thecoder.cafe/i/194114637?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png","isProcessing":false,"align":null,"offset":false}" alt="Diagram showing the three states of a system experiencing a metastable failure: Stable, Vulnerable, and Metastable. Arrows show transitions: increasing load moves the system from Stable to Vulnerable, a trigger moves it from Vulnerable to Metastable, and a strong corrective action (dashed arrow) brings it back to Stable." title="Diagram showing the three states of a system experiencing a metastable failure: Stable, Vulnerable, and Metastable. Arrows show transitions: increasing load moves the system from Stable to Vulnerable, a trigger moves it from Vulnerable to Metastable, and a strong corrective action (dashed arrow) brings it back to Stable." srcset="https://substackcdn.com/image/fetch/$s_!qhkv!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 424w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 848w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 1272w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 1456w" sizes="100vw" class="sizing-normal"/></picture><div></div></div></a><figcaption class="image-caption">The three states of a metastable failure. A system can drift into the vulnerable state unnoticed, and a single trigger is enough to push it into the metastable state it cannot escape on its own.</figcaption></figure></div><p>The most disorienting property of a metastable failure: stopping the trigger doesn’t stop the failure. To understand why, we need to talk about feedback loops.</p><h1 class="header-anchor-post">The Feedback Loop is The Root Cause<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§the-feedback-loop-is-the-root-cause" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.thecoder.cafe/i/194114637/the-feedback-loop-is-the-root-cause" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><p><span>In a previous post on </span><a href="https://read.thecoder.cafe/p/systems-thinking">Systems Thinking Explained</a><span>, we defined a feedback loop as: If </span><code>A</code><span> causes </span><code>B</code><span>, then </span><code>B</code><span> influences </span><code>A</code><span>.</span></p><p><span>A feedback loop is exactly the mechanism that keeps a system </span><strong>stuck in the metastable state</strong><span>. There is always a sustaining effect, a feedback loop, that prevents recovery. The trigger is just what pushes the system over the edge. The loop is what keeps it there. Blaming the trigger is the natural instinct, and almost always the wrong diagnosis.</span></p><p>Let’s discuss a concrete example to make this clear.</p><p>Imagine a web application that queries a database. The database comfortably handles up to 300 QPS. The application retries any query that doesn’t respond within 1 second. The system is running at 280 QPS, healthy and fast, within the database’s capacity.</p><p>Then, a transient network issue occurs for 10 seconds. When the issue is over, all the queued requests flood in at once. The database gets hit with a surge it can’t absorb: latency spikes and queries start timing out. So the application retries them. This doubles the effective load to 560 QPS. The database, already struggling, falls further behind. More timeouts. More retries. The loop is now self-sustaining:</p><blockquote><p><em>High load → Timeouts → Retries → Higher load → More timeouts → More retries</em></p></blockquote><p><span>The transient network issue was fixed minutes ago. Yet, the system is still completely broken. The trigger is gone; </span><strong>the feedback loop is not</strong><span>. The only way out is to dramatically cut the load or disable retries entirely.</span></p><p><span>This is a </span><strong>metastable failure</strong><span>. The system was vulnerable because it was operating close to its </span><strong>hidden capacity</strong><span>. A minor, transient </span><strong>trigger</strong><span> pushed it over the edge and into a self-sustaining failure state it couldn’t escape on its own. The retry mechanism, a feature designed to improve reliability, became the very thing that prevented recovery.</span></p><p><span>This is one example, but the same pattern appears with caches, connection pools, failover logic, and more. The shape is always the same: </span><strong>a feedback loop that turns a temporary problem into a permanent one</strong><span>.</span></p><h1 class="header-anchor-post">Why It’s So Hard To Fix<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§why-its-so-hard-to-fix" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.thecoder.cafe/i/194114637/why-its-so-hard-to-fix" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><p>Two things make metastable failures particularly nasty.</p><ul><li><p><strong>We can be tempted to blame the wrong thing</strong><span>. When an outage happens, the trigger is what’s visible and recent: a spike, a deployment, a hardware fault. It’s the obvious culprit. But the trigger only exposed the problem; it didn’t create it. The sustaining feedback loop was already there, structural and invisible. When analyzing the problem in retrospect, teams focus on the trigger; fixes address the trigger; and the system remains vulnerable to the next one. The authors of the paper observed teams declare a metastable failure “resolved” multiple times before realizing the real cause had never been touched.</span></p></li><li><p><strong>The feedback loop grows stronger with scale</strong><span>. Small-scale tests won’t reveal it. A staging environment running at 10% capacity may handle the same trigger without falling into a metastable state, because the loop isn’t strong enough at that scale to be self-sustaining. This means these failures can slip past even rigorous testing regimes and only manifest in production at full load.</span></p></li></ul><p><span>We defined hidden capacity earlier as the load level below which the system can self-heal from any disruption. It’s different, and always lower, than the advertised capacity. In our example, the numbers make it concrete: the advertised capacity is 300 QPS, but the hidden capacity is only 150 QPS, because retries double the load under failure. The gap between those two numbers is </span><strong>where vulnerability lives</strong><span>.</span></p><p>Measuring the hidden capacity is not straightforward, though. One possible approach is to apply a trigger at a given load level and observe whether the system recovers on its own:</p><ul><li><p>If it does, we are below the hidden capacity.</p></li><li><p>If it doesn’t, we are above it.</p></li></ul><p>We can also estimate it indirectly: in the retry example, retries double the load under failure, so the hidden capacity is roughly half the advertised capacity.</p><h1 class="header-anchor-post">Finding the Loop, Not the Bug<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§finding-the-loop-not-the-bug" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.thecoder.cafe/i/194114637/finding-the-loop-not-the-bug" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><p><strong>Metastable failures are not bugs</strong><span>. We can’t write a unit test that catches them. They are emergent behaviors: properties that arise from the interaction of a system’s components under specific conditions, not logic errors in any individual component. No single piece of code is buggy, no single configuration is wrong. The failure is a consequence of how everything fits together under load.</span></p><p><span>This changes how we need to think about them. The right question after an outage is not “</span><em>What failed?</em><span>” but “</span><em>What loop sustained it?</em><span>” And before an outage, the danger is not having bugs; it’s optimizing so aggressively for efficiency that </span><strong>we push the system deeper into the vulnerable state without realizing it</strong><span>.</span></p><p>Retries, caches, failover logic, connection pools: these are all features that improve reliability in the common case. They are also, under the right conditions, the sustaining mechanisms of metastable failures. The same design decision that makes a system more resilient in normal operation can also prevent it from recovering when things go wrong.</p><h1 class="header-anchor-post">What Can We Do About It?<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§what-can-we-do-about-it" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.thecoder.cafe/i/194114637/what-can-we-do-about-it" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><p>The paper describes several approaches to reduce the risk of metastable failures:</p><ul><li><p><strong>Retry budgets and circuit breakers</strong><span>: Instead of retrying indefinitely, cap the total number of retries in flight at any given time. This directly weakens the feedback loop by limiting work amplification.</span></p></li><li><p><strong>LIFO scheduling under overload</strong><span>: Counterintuitively, switching from FIFO to LIFO when the system is overloaded allows some requests to complete within their deadline, preserving goodput instead of letting every request time out.</span></p><blockquote><p><strong>NOTE</strong><span>: </span><em><span>I already wrote a post about that approach in </span><a href="https://read.thecoder.cafe/p/adaptive-lifo">Adaptive LIFO</a><span>.</span></em></p></blockquote></li><li><p><strong>Fast error paths</strong><span>: Success paths are heavily optimized, but error paths often aren’t. An expensive error path (stack traces, DNS lookups, disk writes) under high failure rates can itself become a sustaining mechanism. Optimizing error paths reduces this risk.</span></p></li><li><p><strong>Read-through caches over look-aside caches</strong><span>: A read-through cache (where the cache itself fetches missing data from the database) can continue filling itself even when the application has given up on a request, steadily increasing the hit rate and helping the system recover. A look-aside cache (where the application is responsible for populating the cache) can’t.</span></p></li><li><p><strong>Production stress testing</strong><span>: Small-scale tests won’t reveal metastable failures. Testing against a portion of production traffic, with engineers ready to intervene, is the most reliable way to surface them.</span></p></li></ul><p>A note of humility from the paper: there is no systematic solution yet. These are ad-hoc mitigations developed in response to known failures. Detecting vulnerable states before they collapse remains an open problem.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://read.thecoder.cafe" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!ZoDz!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 424w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 848w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 1272w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!ZoDz!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png" width="449" height="224.5" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/c5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":600,"width":1200,"resizeWidth":449,"bytes":82853,"alt":"The Coder Cafe: Learn One Concept With Your Coffee.","title":"The Coder Cafe: Learn One Concept With Your Coffee.","type":"image/png","href":"https://read.thecoder.cafe","belowTheFold":true,"topImage":false,"internalRedirect":"https://www.thecoder.cafe/i/151119215?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png","isProcessing":false,"align":null,"offset":false}" alt="The Coder Cafe: Learn One Concept With Your Coffee." title="The Coder Cafe: Learn One Concept With Your Coffee." srcset="https://substackcdn.com/image/fetch/$s_!ZoDz!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 424w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 848w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 1272w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><div class="subscription-widget-wrap"><div class="subscription-widget show-subscribe"><div class="preamble"><p>AI is getting better every day. Are you? At The Coder Cafe, we serve fundamental concepts to make you an engineer that AI won’t replace. Written by a Google SWE, trusted by thousands of engineers worldwide.</p></div><div data-component-name="SubscribeWidget" class="subscribe-widget"><div class="pencraft pc-display-flex pc-justifyContent-center pc-reset"><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe-widget-preamble"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><input type="hidden" name="source_post_id" value="194114637"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div><h1 class="header-anchor-post">Summary<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§summary" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.thecoder.cafe/i/194114637/summary" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><ul><li><p>A distributed system can pass through three states: stable, vulnerable, and metastable. The vulnerable state looks healthy, but it isn’t.</p></li><li><p>The threshold between stable and vulnerable is invisible. Systems can operate in the vulnerable state for months without any sign of trouble.</p></li><li><p>When a trigger pushes a vulnerable system into a metastable failure, a feedback loop sustains the failure even after the trigger is gone.</p></li><li><p>The trigger is not the root cause. The feedback loop is. Fixing the trigger leaves the system vulnerable to the next one.</p></li><li><p>Reliability features like retries and caches can become the sustaining mechanism of a metastable failure under the right conditions.</p></li><li><p>Metastable failures are emergent behaviors, not bugs. We can’t unit test for them, and optimizing for efficiency makes them more likely.</p></li><li><p>Mitigations exist (retry budgets, circuit breakers, LIFO scheduling, fast error paths), but they are all ad-hoc responses to known failures. Detecting vulnerable states before they collapse remains an open problem.</p></li></ul><h1 class="header-anchor-post">Resources<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§resources" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.thecoder.cafe/i/194114637/resources" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><h2 class="header-anchor-post"><span>More From the </span><em><a href="https://read.thecoder.cafe/t/reliability">Reliability</a></em><span> Category</span><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§more-from-the-reliability-category" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.thecoder.cafe/i/194114637/more-from-the-reliability-category" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><ul><li><p><a href="https://read.thecoder.cafe/p/resilient-fault-tolerant-robust-reliable">Resilient, Fault-tolerant, Robust, or Reliable?</a></p></li><li><p><a href="https://read.thecoder.cafe/p/adaptive-lifo">Adaptive LIFO</a></p></li><li><p><a href="https://read.thecoder.cafe/p/fail-open-fail-closed">Fail Open vs. Fail Closed</a></p></li></ul><h2 class="header-anchor-post">Sources<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§sources" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.thecoder.cafe/i/194114637/sources" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><ul><li><p><a href="https://sigops.org/s/conferences/hotos/2021/papers/hotos21-s11-bronson.pdf">Metastable Failures in Distributed Systems</a></p></li></ul><h2 class="header-anchor-post">Explore Further<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§explore-further" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.thecoder.cafe/i/194114637/explore-further" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><ul><li><p><a href="https://brooker.co.za/blog/2021/05/24/metastable.html">Metastability and Distributed Systems</a></p></li></ul></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (12)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">12</div></button></div><button tabindex="0" type="button" aria-label="View comments (2)" data-href="https://read.thecoder.cafe/p/metastable-failures/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg><div class="label">2</div></button><button tabindex="0" type="button" id="radix-P0-56" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">1</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-flex-start pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-left"><path d="m12 19-7-7 7-7"></path><path d="M19 12H5"></path></svg>Previous</button></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="visibility-check"></div><div id="discussion" class="pencraft pc-display-flex pc-flexDirection-column pc-gap-16 pc-paddingTop-32 pc-paddingBottom-32 pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-32 pc-reset container"><h4 class="pencraft pc-reset line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">2 Comments</h4></div><div id="substack-comments" class="single-post-section comments-section"><div class="container"><div class="visibility-check"></div><div data-test-id="comment-input" class="pencraft pc-display-flex pc-reset flex-grow-rzmknG"><form class="comment-input-form form-CkZ7Kt"><div style="--scale:32px;" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><div style="--scale:32px;" title="User" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-flexDirection-column pc-boxShadow-xs pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-sm"><textarea name="body" placeholder="Write a comment..." aria-label="Write a comment..." rows="4" class="pencraft input-qHk4bN autogrowing-_ipn9Y textarea-GbEjRX inputText-pV_yWb unstyled-IJLUJU"></textarea></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></form></div><div class="comment-list post-page-root-comment-list"><div class="comment-list-items"><div class="comment"><div id="comment-266416976" class="comment-anchor"></div><div id="comment-266416976-reply" class="comment-anchor"></div><div role="article" aria-label="Comment by Tushar Raturi" class="pencraft pc-display-flex pc-gap-12 pc-paddingBottom-12 pc-reset comment-content"><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><a href="https://substack.com/profile/508383034-tushar-raturi?utm_source=comment" aria-label="View Tushar Raturi's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:32px;" tabindex="0" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6"><div style="--scale:32px;" title="Tushar Raturi" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!uXtH!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2027700-2e2c-410b-bc1a-3b249f88ed17_320x322.png 1x, https://substackcdn.com/image/fetch/$s_!uXtH!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2027700-2e2c-410b-bc1a-3b249f88ed17_320x322.png 2x, https://substackcdn.com/image/fetch/$s_!uXtH!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2027700-2e2c-410b-bc1a-3b249f88ed17_320x322.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!uXtH!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2027700-2e2c-410b-bc1a-3b249f88ed17_320x322.png" alt="Tushar Raturi's avatar" srcset="https://substackcdn.com/image/fetch/$s_!uXtH!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2027700-2e2c-410b-bc1a-3b249f88ed17_320x322.png 1x, https://substackcdn.com/image/fetch/$s_!uXtH!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2027700-2e2c-410b-bc1a-3b249f88ed17_320x322.png 2x, https://substackcdn.com/image/fetch/$s_!uXtH!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2027700-2e2c-410b-bc1a-3b249f88ed17_320x322.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-4 pc-reset"><div class="pencraft pc-display-flex pc-minWidth-0 pc-gap-8 pc-alignItems-center pc-justifyContent-space-between pc-reset line-height-20-t4M0El font-text-qe4AeH size-15-Psle70 weight-regular-mUq6Gb"><div class="pencraft pc-display-flex pc-minWidth-0 pc-gap-8 pc-alignItems-center pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-gap-6 pc-reset color-primary-zABazT line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb reset-IxiVJZ comment-author-name"><div class="pencraft pc-display-flex pc-gap-4 pc-minWidth-0 pc-alignItems-center pc-reset"><span class="pencraft pc-reset ellipsis-Jb8om8 line-height-20-t4M0El font-text-qe4AeH size-15-Psle70 weight-medium-fw81nC reset-IxiVJZ"><span style="min-width:0;" data-state="closed"><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a href="https://substack.com/profile/508383034-tushar-raturi?utm_source=substack-feed-item" showBack data-native="true" class="link-LIBpto">Tushar Raturi</a></span></span></span></div></div><span style="min-width:0;" data-state="closed"></span><a data-native="true" href="https://read.thecoder.cafe/p/metastable-failures/comment/266416976" rel="nofollow" title="May 28, 2026, 6:55 AM" class="pencraft pc-reset color-secondary-ls1g8s decoration-hover-underline-ClDVRM reset-IxiVJZ"><span class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb nowrap-QngyoB reset-IxiVJZ">May 28</span></a><span title="May 28, 2026, 6:55 AM" class="pencraft pc-reset cursor-pointer-LYORKw color-tertiary-ePI2vV line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr font-style-italic-I3_Jdy weight-regular-mUq6Gb decoration-hover-underline-ClDVRM reset-IxiVJZ">Edited</span></div></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><button tabindex="-1" type="button" class="pencraft pc-display-flex pc-gap-4 pc-height-20 pc-paddingLeft-6 pc-paddingRight-6 pc-paddingTop-2 pc-paddingBottom-2 pc-alignItems-center pc-reset font-text-qe4AeH size-11-NuY2Zx weight-medium-fw81nC pencraft tag-XbOVLt theme_accent-Y2sqZY priority_secondary-outline-RpooJS pencraft pc-display-flex pc-height-20 pc-paddingLeft-6 pc-paddingRight-6 pc-paddingTop-2 pc-paddingBottom-2 pc-gap-4 pc-alignItems-center pc-reset cursor-inherit-LxLBJ6 pc-borderRadius-xs font-text-qe4AeH size-11-NuY2Zx weight-medium-fw81nC"><div class="pencraft pc-display-flex pc-alignItems-center pc-reset leading-mI5Ihl fillIcon-dQ0mii"><svg aria-hidden="true" style="height:14px;width:14px;" width="14" height="14" viewBox="0 0 20 20" fill="var(--color-fg-primary)" stroke-width="2.5" stroke="#000" xmlns="http://www.w3.org/2000/svg"><g><path stroke="none" d="M9.99915 16.7256C9.90515 16.7256 9.79102 16.692 9.65674 16.6249C9.52246 16.5622 9.3949 16.4906 9.27405 16.41C8.02974 15.6044 6.94657 14.7584 6.02454 13.8722C5.10697 12.9815 4.3953 12.0662 3.88953 11.1262C3.38375 10.1818 3.13086 9.23067 3.13086 8.27283C3.13086 7.63725 3.23157 7.05762 3.43298 6.53394C3.63888 6.01025 3.92086 5.55819 4.27893 5.17773C4.64148 4.79728 5.05774 4.50635 5.52771 4.30493C6.00216 4.09904 6.51241 3.99609 7.05847 3.99609C7.73433 3.99609 8.31844 4.16618 8.81079 4.50635C9.30762 4.84652 9.70374 5.28963 9.99915 5.83569C10.299 5.28516 10.6951 4.84204 11.1875 4.50635C11.6843 4.16618 12.2707 3.99609 12.9465 3.99609C13.4836 3.99609 13.9894 4.09904 14.4639 4.30493C14.9428 4.50635 15.3613 4.79728 15.7194 5.17773C16.0774 5.55819 16.3572 6.01025 16.5586 6.53394C16.7645 7.05762 16.8674 7.63725 16.8674 8.27283C16.8674 9.23067 16.6145 10.1818 16.1088 11.1262C15.603 12.0662 14.8891 12.9815 13.967 13.8722C13.0495 14.7584 11.9708 15.6044 10.731 16.41C10.6056 16.4906 10.4758 16.5622 10.3416 16.6249C10.2118 16.692 10.0976 16.7256 9.99915 16.7256Z"></path></g></svg></div>Liked by Teiva Harsanyi</button></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"></div><div class="pencraft pc-display-flex pc-reset comment-menu-trigger-container triggerContainer-eX588u"><button tabindex="0" type="button" aria-label="Ellipsis" id="radix-P0-80" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft trigger-j08Uop iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_quaternary-kpMibu"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-ellipsis"><circle cx="12" cy="12" r="1"></circle><circle cx="19" cy="12" r="1"></circle><circle cx="5" cy="12" r="1"></circle></svg></button></div></div><div class="comment-body expanded"><p><span>That was a great read…and I realised something.</span></p><p><span>The individual human system (biologically) is a one of the largest distributed system. If there is a “metastable failure” in this system, there is a disease. Sometimes the human system recovers…but sometimes it cant. (Note that I am not talking about the natural lifespan ending but chronic or external diseases/issues).</span></p><p><span>How has this system bettered over time? Through evolution. Of course, now, the system is so complex that it is generally adaptive even without evolution.</span></p><p><span>Keeping that in mind, Evolutionary/Genetic algorithms, Generative Adversarial Networks, and Reinforcement learning seem to be some things which could work for this use case.</span></p></div><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-8 pc-justifyContent-flex-start pc-alignItems-center pc-reset comment-actions withShareButton-hQzuEn"><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a class="pencraft pc-reset link-_X6et2 link-LIBpto"><div class="pencraft pc-display-flex pc-gap-6 pc-alignItems-center pc-reset"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="var(--color-fg-secondary-themed)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA comment-actions-label">Reply</div></div></a></span><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a class="pencraft pc-reset link-_X6et2 link-LIBpto"><div class="pencraft pc-display-flex pc-gap-6 pc-alignItems-center pc-reset"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="var(--color-fg-secondary-themed)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA comment-actions-label">Share</div></div></a></span></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></div><div class="more-replies-container"><a href="https://read.thecoder.cafe/p/metastable-failures/comment/266416976" class="more-replies">1 reply by Teiva Harsanyi</a></div></div></div></div><a href="https://read.thecoder.cafe/p/metastable-failures/comments" class="more-comments">1 more comment...</a></div></div></div><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-98-content-top" data-state="active" id="radix-P0-98-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-98-content-new" data-state="inactive" id="radix-P0-98-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-98-content-community" data-state="inactive" id="radix-P0-98-trigger-community" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Discussions</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer themed-background"><div class="container"><div class="footer-terms"><span>© 2026 Teiva Harsanyi</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&utm_medium=web&utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="SiMLZxbT83ypi3YnctfvQQ">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
|
||
</div>
|
||
|
||
|
||
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
|
||
<script nonce="SiMLZxbT83ypi3YnctfvQQ">
|
||
window.Sentry && window.Sentry.onLoad(function() {
|
||
window.Sentry.init({
|
||
environment: window._preloads.sentry_environment,
|
||
dsn: window._preloads.sentry_dsn,
|
||
})
|
||
})
|
||
</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script nonce="SiMLZxbT83ypi3YnctfvQQ">window._preloads = JSON.parse("{\"cspNonce\":\"SiMLZxbT83ypi3YnctfvQQ\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://read.thecoder.cafe\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":3529974,\"byline_images_enabled\":true,\"bylines_enabled\":true,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Teiva Harsanyi\",\"cover_photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/5cc959dd-e722-43f8-87a1-875c5e6c3cdd_1200x600.png\",\"created_at\":\"2024-08-11T19:08:26.110Z\",\"custom_domain_optional\":false,\"custom_domain\":\"read.thecoder.cafe\",\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":null,\"default_show_guest_bios\":true,\"email_banner_url\":null,\"email_from_name\":\"The Coder Cafe\",\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":[\"foo https://github.com\"],\"free_subscription_benefits\":[\"\u2705 Full content.\"],\"ga_pixel_id\":null,\"google_site_verification_token\":null,\"google_tag_manager_token\":\"GTM-KDC62RGC\",\"hero_image\":null,\"hero_text\":\"AI is getting better every day. Are you? At The Coder Cafe, we serve fundamental concepts to make you an engineer that AI won't replace. Written by a Google SWE, trusted by thousands of engineers worldwide.\",\"hide_intro_subtitle\":false,\"hide_intro_title\":true,\"hide_podcast_feed_link\":false,\"homepage_type\":\"newspaper\",\"id\":2874913,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":\"https://substackcdn.com/image/fetch/$s_!xko8!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09e5dcc0-f9fe-47f0-a81e-0148d03f7535_1344x256.png\",\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!OZXv!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png\",\"minimum_group_size\":2,\"moderation_enabled\":true,\"name\":\"The Coder Cafe\",\"paid_subscription_benefits\":[\"\u2705 Full content (same as free).\",\"\uD83E\uDEF6 If you enjoy my content, this is the simplest way to show your support.\"],\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"enabled\",\"paywall_free_trial_enabled\":false,\"podcast_art_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/5144afcf-c93d-4fa6-9779-938926892a50_1400x1400.png\",\"paid_podcast_episode_art_url\":null,\"podcast_byline\":\"Teiva Harsanyi\",\"podcast_description\":\"Missing direction in your tech career? At The Coder Cafe, we serve timeless concepts with your coffee to help you master the fundamentals. Written by a Google SWE and trusted by 3,000+ readers, we support your growth as an engineer, one coffee at a time.\",\"podcast_enabled\":true,\"podcast_feed_url\":null,\"podcast_title\":\"The Coder Cafe Podcast\",\"post_preview_limit\":0,\"primary_user_id\":3529974,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"thecodercafe\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#FF6719\",\"theme_var_color_links\":true,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":null,\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":[{\"id\":\"monthly5usd\",\"object\":\"plan\",\"active\":true,\"aggregate_usage\":null,\"amount\":500,\"amount_decimal\":\"500\",\"billing_scheme\":\"per_unit\",\"created\":1727988327,\"currency\":\"usd\",\"interval\":\"month\",\"interval_count\":1,\"livemode\":true,\"metadata\":{\"substack\":\"yes\"},\"meter\":null,\"nickname\":\"$5 a month\",\"product\":\"prod_QxrdKSGKjk6zgg\",\"tiers_mode\":null,\"transform_usage\":null,\"trial_period_days\":null,\"usage_type\":\"licensed\",\"currency_options\":{\"aud\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":800,\"unit_amount_decimal\":\"800\"},\"brl\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":2600,\"unit_amount_decimal\":\"2600\"},\"cad\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":800,\"unit_amount_decimal\":\"800\"},\"chf\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":500,\"unit_amount_decimal\":\"500\"},\"dkk\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":3300,\"unit_amount_decimal\":\"3300\"},\"eur\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":500,\"unit_amount_decimal\":\"500\"},\"gbp\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":400,\"unit_amount_decimal\":\"400\"},\"inr\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":48000,\"unit_amount_decimal\":\"48000\"},\"jpy\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":800,\"unit_amount_decimal\":\"800\"},\"mxn\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":9000,\"unit_amount_decimal\":\"9000\"},\"nok\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":4800,\"unit_amount_decimal\":\"4800\"},\"nzd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":900,\"unit_amount_decimal\":\"900\"},\"pln\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":1900,\"unit_amount_decimal\":\"1900\"},\"sek\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":4800,\"unit_amount_decimal\":\"4800\"},\"usd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":500,\"unit_amount_decimal\":\"500\"}}},{\"id\":\"yearly50usd\",\"object\":\"plan\",\"active\":true,\"aggregate_usage\":null,\"amount\":5000,\"amount_decimal\":\"5000\",\"billing_scheme\":\"per_unit\",\"created\":1727988328,\"currency\":\"usd\",\"interval\":\"year\",\"interval_count\":1,\"livemode\":true,\"metadata\":{\"substack\":\"yes\"},\"meter\":null,\"nickname\":\"$50 a year\",\"product\":\"prod_Qxrd6wZbT5KEB1\",\"tiers_mode\":null,\"transform_usage\":null,\"trial_period_days\":null,\"usage_type\":\"licensed\",\"currency_options\":{\"aud\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":7500,\"unit_amount_decimal\":\"7500\"},\"brl\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":25500,\"unit_amount_decimal\":\"25500\"},\"cad\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":7500,\"unit_amount_decimal\":\"7500\"},\"chf\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":4100,\"unit_amount_decimal\":\"4100\"},\"dkk\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":32500,\"unit_amount_decimal\":\"32500\"},\"eur\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":4400,\"unit_amount_decimal\":\"4400\"},\"gbp\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":3800,\"unit_amount_decimal\":\"3800\"},\"inr\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":478000,\"unit_amount_decimal\":\"478000\"},\"jpy\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":8000,\"unit_amount_decimal\":\"8000\"},\"mxn\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":87000,\"unit_amount_decimal\":\"87000\"},\"nok\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":48000,\"unit_amount_decimal\":\"48000\"},\"nzd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":9000,\"unit_amount_decimal\":\"9000\"},\"pln\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":19000,\"unit_amount_decimal\":\"19000\"},\"sek\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":48000,\"unit_amount_decimal\":\"48000\"},\"usd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":5000,\"unit_amount_decimal\":\"5000\"}}}],\"stripe_user_id\":\"acct_1PmhSLG30FjyGqWC\",\"stripe_country\":\"FR\",\"stripe_publishable_key\":\"pk_live_51PmhSLG30FjyGqWCDIwENYFQnxZpHzgdjt22aQKf3XzEWxZObvoP99s8yasLT515yPX36a0vTY7wV4vGqJLxGhRS00ALJBivGg\",\"stripe_platform_account\":\"US\",\"automatic_tax_enabled\":false,\"author_name\":\"Teiva Harsanyi\",\"author_handle\":\"teivah\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!XdES!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png\",\"author_bio\":\"Software Engineer at Google, Author, Speaker.\",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":\"#8d493a\",\"web_bg_color\":\"#f8ede3\",\"cover_bg_color\":\"#f8ede3\",\"publication_id\":2874913,\"color_links\":null,\"font_preset_heading\":null,\"font_preset_body\":null,\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"newspaper\",\"home_posts\":\"custom\",\"home_show_top_posts\":true,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":{\"layout\":\"stacked\",\"navStyle\":\"text\",\"navPosition\":\"bottom\",\"backgroundColor\":null,\"wordmarkLogoSize\":36},\"custom_footer\":{\"layout\":\"default\",\"backgroundColor\":null,\"publicationNameSize\":50,\"showPublicationName\":true,\"publicationNameStyle\":\"text\",\"publicationNamePosition\":\"bottom\",\"publicationNameBackgroundColor\":null},\"social_media_links\":{\"bluesky\":null},\"font_options\":null,\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":{\"first_thread_email_sent_at\":null,\"activated_at\":\"2024-10-07T09:42:26.823+00:00\",\"reader_thread_notifications_enabled\":true,\"create_thread_minimum_role\":\"free_subscriber\",\"photo_replies_enabled\":true,\"boost_free_subscriber_chat_preview_enabled\":false,\"push_suppression_enabled\":false},\"default_group_coupon_percent_off\":null,\"default_group_coupon_include_founding\":null,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2024-08-31T08:25:06.610Z\",\"has_podcast\":false,\"has_free_podcast\":false,\"has_subscriber_only_podcast\":false,\"has_community_content\":true,\"rankingDetail\":\"Launched 2 years ago\",\"rankingDetailFreeIncluded\":\"Thousands of subscribers\",\"rankingDetailOrderOfMagnitude\":1,\"rankingDetailFreeIncludedOrderOfMagnitude\":1000,\"rankingDetailFreeSubscriberCount\":\"Over 8,000 subscribers\",\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u062A\u0645 \u0627\u0644\u0625\u0637\u0644\u0627\u0642 2 years ago\"},\"ca\":{\"rankingDetail\":\"S\u2019ha llan\u00E7at fa 2 anys\"},\"da\":{\"rankingDetail\":\"Lancering 2 \u00E5r\"},\"de\":{\"rankingDetail\":\"vor 2 Jahren gelauncht\"},\"es\":{\"rankingDetail\":\"Lanzado hace 2 a\u00F1os\"},\"fr\":{\"rankingDetail\":\"Lanc\u00E9 il y a 2 ann\u00E9es\"},\"id\":{\"rankingDetail\":\"Diluncurkan 2 tahun yang lalu\"},\"ja\":{\"rankingDetail\":\"\u958B\u59CB\u65E5 2\u5E74\u524D\"},\"ko\":{\"rankingDetail\":\"2\uB144 \uC804\uC5D0 \uCD9C\uC2DC\"},\"nb\":{\"rankingDetail\":\"Lansert 2 \u00E5r\"},\"nl\":{\"rankingDetail\":\"Gelanceerd 2 jaar geleden\"},\"pl\":{\"rankingDetail\":\"Uruchomiono 2 lat temu\"},\"pt\":{\"rankingDetail\":\"Lan\u00E7ado 2 anos\"},\"pt-br\":{\"rankingDetail\":\"Lan\u00E7ado 2 anos\"},\"en-gb\":{\"rankingDetail\":\"Launched 2 years ago\"},\"it\":{\"rankingDetail\":\"Lanciato 2 anni\"},\"tr\":{\"rankingDetail\":\"2 y\u0131l ba\u015Flat\u0131ld\u0131\"},\"sv\":{\"rankingDetail\":\"Lanserad 2 \u00E5r sedan\"},\"fi\":{\"rankingDetail\":\"Launched 2 vuotta\"},\"is\":{\"rankingDetail\":\"Launched 2 \u00E1r\"},\"zh-hans\":{\"rankingDetail\":\"\u4E8E2 \u5E74\u524D\u4E0A\u7EBF\"},\"zh-hant\":{\"rankingDetail\":\"\u65BC 2 \u5E74\u524D \u63A8\u51FA\"},\"en\":{\"rankingDetail\":\"Launched 2 years ago\"}},\"freeSubscriberCount\":\"8,000\",\"freeSubscriberCountOrderOfMagnitude\":\"8.8K+\",\"author_bestseller_tier\":0,\"author_badge\":{\"type\":\"subscriber\",\"tier\":1,\"accent_colors\":null},\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":true,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":\"2025-02-04T09:23:51.746Z\",\"primary_profile_name\":\"Teiva Harsanyi\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!XdES!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png\",\"no_follow\":false,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"free\",\"sections\":[{\"id\":206570,\"created_at\":\"2025-04-02T23:25:22.621Z\",\"updated_at\":\"2026-04-24T13:10:21.528Z\",\"publication_id\":2874913,\"name\":\"Concepts\",\"description\":\"Learn one concept with your coffee.\",\"slug\":\"concepts\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":0,\"port_status\":\"success\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/d2946fd2-0953-4810-ad71-4d4cd4769c9f_1280x1280.png\",\"hide_from_navbar\":false,\"email_from_name\":\"The Coder Cafe\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"showLinks\":[],\"unifiedPodcastSettings\":null,\"pageTheme\":{\"id\":63961,\"publication_id\":2874913,\"section_id\":206570,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"left\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":265954,\"created_at\":\"2025-08-25T21:40:10.969Z\",\"updated_at\":\"2026-04-24T13:10:21.528Z\",\"publication_id\":2874913,\"name\":\"The Coding Corner\",\"description\":\"A corner of The Coder Cafe where we build real-world systems together.\",\"slug\":\"coding-corner\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":1,\"port_status\":\"success\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/8c694f3f-93da-43f8-b7e3-76093caebaf7_1280x1280.png\",\"hide_from_navbar\":false,\"email_from_name\":\"The Coder Cafe\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"showLinks\":[],\"unifiedPodcastSettings\":null,\"pageTheme\":{\"id\":63964,\"publication_id\":2874913,\"section_id\":265954,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"left\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":206264,\"created_at\":\"2025-04-02T07:57:04.764Z\",\"updated_at\":\"2026-04-24T13:10:21.528Z\",\"publication_id\":2874913,\"name\":\"Lattes & Stories\",\"description\":\"Personal reflections and stories.\",\"slug\":\"lattes-stories\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":2,\"port_status\":\"success\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/7763631f-c32d-4502-85fd-889c6af7a38b_1280x1280.png\",\"hide_from_navbar\":false,\"email_from_name\":\"The Coder Cafe\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"showLinks\":[],\"unifiedPodcastSettings\":null,\"pageTheme\":{\"id\":35705,\"publication_id\":2874913,\"section_id\":206264,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"left\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":206576,\"created_at\":\"2025-04-02T23:45:35.632Z\",\"updated_at\":\"2026-04-24T13:10:21.528Z\",\"publication_id\":2874913,\"name\":\"Announcements\",\"description\":\"The Coder Cafe news and updates.\",\"slug\":\"announcements\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":3,\"port_status\":\"success\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/507685d6-2d67-472e-9d92-b8a9cee9d32d_1280x1280.png\",\"hide_from_navbar\":true,\"email_from_name\":\"The Coder Cafe\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"showLinks\":[],\"unifiedPodcastSettings\":null,\"pageTheme\":{\"id\":35704,\"publication_id\":2874913,\"section_id\":206576,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"left\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}],\"podcastTabInfo\":null,\"multipub_migration\":null,\"navigationBarItems\":[{\"id\":\"b0b92987-ecae-4da8-8b0d-ba6927d07da9\",\"publication_id\":2874913,\"sibling_rank\":0,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":true,\"standard_key\":\"about\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":null},{\"id\":\"7bc72eeb-bfbb-4b46-a775-0d4c4379931b\",\"publication_id\":2874913,\"sibling_rank\":0,\"link_title\":null,\"link_url\":null,\"section_id\":206570,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":{\"id\":206570,\"slug\":\"concepts\",\"name\":\"Concepts\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/d2946fd2-0953-4810-ad71-4d4cd4769c9f_1280x1280.png\",\"newsletter_enabled\":null}},{\"id\":\"78f73f88-a221-4515-bb1b-3b23cc89fb1a\",\"publication_id\":2874913,\"sibling_rank\":1,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":true,\"standard_key\":\"notes\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":null},{\"id\":\"646cbafc-c08e-4224-9429-438708dd8e74\",\"publication_id\":2874913,\"sibling_rank\":1,\"link_title\":null,\"link_url\":null,\"section_id\":265954,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":{\"id\":265954,\"slug\":\"coding-corner\",\"name\":\"The Coding Corner\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/8c694f3f-93da-43f8-b7e3-76093caebaf7_1280x1280.png\",\"newsletter_enabled\":null}},{\"id\":\"992833c7-c6f1-4f54-8aa1-16f5d1557dbc\",\"publication_id\":2874913,\"sibling_rank\":2,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":true,\"standard_key\":\"leaderboard\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":null},{\"id\":\"836460c4-443c-4e05-be0a-1fcff6068da7\",\"publication_id\":2874913,\"sibling_rank\":2,\"link_title\":null,\"link_url\":null,\"section_id\":206264,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":{\"id\":206264,\"slug\":\"lattes-stories\",\"name\":\"Lattes & Stories\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/7763631f-c32d-4502-85fd-889c6af7a38b_1280x1280.png\",\"newsletter_enabled\":null}},{\"id\":\"a2008385-c99e-4b57-b5ab-33ef7e93da62\",\"publication_id\":2874913,\"sibling_rank\":3,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":true,\"standard_key\":\"chat\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":null},{\"id\":\"ce0f1343-d95d-4c08-ae30-8039b7eb3339\",\"publication_id\":2874913,\"sibling_rank\":3,\"link_title\":\"The Reading Room\",\"link_url\":\"\",\"section_id\":null,\"post_id\":195340886,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":{\"id\":195340886,\"slug\":\"the-reading-room\",\"title\":\"The Reading Room\",\"type\":\"page\"},\"section\":null},{\"id\":\"4f5c134a-3b17-44fa-b5c6-7b7866745b88\",\"publication_id\":2874913,\"sibling_rank\":4,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":true,\"standard_key\":\"archive\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":null},{\"id\":\"10049037-0f4b-4a4d-9bef-4b82564a7f4d\",\"publication_id\":2874913,\"sibling_rank\":4,\"link_title\":null,\"link_url\":null,\"section_id\":206576,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":{\"id\":206576,\"slug\":\"announcements\",\"name\":\"Announcements\",\"hide_from_navbar\":true,\"is_podcast\":false,\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/507685d6-2d67-472e-9d92-b8a9cee9d32d_1280x1280.png\",\"newsletter_enabled\":null}},{\"id\":\"4523b9f3-1307-41c2-9584-c783218eab1f\",\"publication_id\":2874913,\"sibling_rank\":5,\"link_title\":\"Sponsorships\",\"link_url\":\"\",\"section_id\":null,\"post_id\":175570635,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":{\"id\":175570635,\"slug\":\"sponsorships\",\"title\":\"Sponsorships\",\"type\":\"page\"},\"section\":null},{\"id\":\"91158aac-92a7-4acc-b03b-984146823ff8\",\"publication_id\":2874913,\"sibling_rank\":6,\"link_title\":\"\uD83D\uDCCC Index\",\"link_url\":\"\",\"section_id\":null,\"post_id\":148329993,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":{\"id\":148329993,\"slug\":\"all\",\"title\":\"\uD83D\uDCCC Index\",\"type\":\"page\"},\"section\":null},{\"id\":\"a7682684-4411-426c-bfd4-d3bd6e76f298\",\"publication_id\":2874913,\"sibling_rank\":7,\"link_title\":\"\uD83D\uDCAC Discord\",\"link_url\":\"https://discord.thecoder.cafe/\",\"section_id\":null,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":null}],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Teiva Harsanyi\",\"handle\":\"teivah\",\"role\":\"admin\",\"owner\":true,\"user_id\":3529974,\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/e800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png\",\"bio\":\"Software Engineer at Google, Author, Speaker.\",\"status\":{\"bestsellerTier\":null,\"subscriberTier\":1,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"subscriber\",\"tier\":1,\"accent_colors\":null},\"subscriber\":null}}],\"threads_v2_enabled\":false,\"viralGiftsConfig\":{\"id\":\"8d7eab1e-f2fd-4f88-81ae-a4bc881ecfe1\",\"publication_id\":2874913,\"enabled\":false,\"gifts_per_user\":5,\"gift_length_months\":1,\"send_extra_gifts\":true,\"message\":\"My personal Substack\",\"created_at\":\"2024-08-11T19:45:24.992805+00:00\",\"updated_at\":\"2024-08-11T19:45:24.992805+00:00\",\"days_til_invite\":14,\"send_emails\":true,\"show_link\":null},\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"iap_advanced_plans\":[{\"sku\":\"IgUNwqBCpv3TOpOTBM\",\"publication_id\":\"2874913\",\"is_active\":true,\"price_base_units\":7000,\"currency_alpha3\":\"usd\",\"period\":\"year\",\"created_at\":\"2025-08-24T10:49:51.822Z\",\"updated_at\":\"2025-08-24T10:49:51.822Z\",\"id\":\"618262\",\"payout_amount_base_units\":500,\"alternate_currencies\":{\"aud\":11000,\"brl\":38000,\"cad\":10000,\"chf\":6000,\"dkk\":45000,\"eur\":6000,\"gbp\":5500,\"mxn\":130500,\"nok\":70500,\"nzd\":12000,\"pln\":25500,\"sek\":66500},\"is_founding\":false,\"display_name\":\"The Coder Cafe (Yearly)\",\"display_price\":\"$70\"},{\"sku\":\"tCOo7rC186oarRCxVv\",\"publication_id\":\"2874913\",\"is_active\":true,\"price_base_units\":700,\"currency_alpha3\":\"usd\",\"period\":\"month\",\"created_at\":\"2025-08-24T10:49:51.730Z\",\"updated_at\":\"2025-08-24T10:49:51.730Z\",\"id\":\"618261\",\"payout_amount_base_units\":50,\"alternate_currencies\":{\"aud\":1100,\"brl\":3800,\"cad\":1000,\"chf\":600,\"dkk\":4500,\"eur\":600,\"gbp\":600,\"mxn\":13500,\"nok\":7500,\"nzd\":1200,\"pln\":2600,\"sek\":7000},\"is_founding\":false,\"display_name\":\"The Coder Cafe (Monthly)\",\"display_price\":\"$7\"}],\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://read.thecoder.cafe\",\"hostname\":\"read.thecoder.cafe\",\"is_on_substack\":false,\"show_links\":[{\"id\":26263,\"publication_id\":2874913,\"section_id\":null,\"url\":\"https://open.spotify.com/show/33C6cekh9H6Hwp3FSdnbVi\",\"platform\":\"spotify\"},{\"id\":26264,\"publication_id\":2874913,\"section_id\":null,\"url\":\"https://open.spotify.com/show/33C6cekh9H6Hwp3FSdnbVi\",\"platform\":\"spotify_for_paid_users\"}],\"spotify_podcast_settings\":{\"id\":24107,\"publication_id\":2874913,\"section_id\":null,\"spotify_uri\":\"spotify:show:33C6cekh9H6Hwp3FSdnbVi\",\"spotify_podcast_title\":null,\"created_at\":\"2024-10-18T14:06:12.070Z\",\"updated_at\":\"2024-10-18T21:44:20.318Z\",\"currently_published_on_spotify\":true,\"spotify_show_url\":\"https://open.spotify.com/show/33C6cekh9H6Hwp3FSdnbVi\"},\"unified_podcast_settings\":null,\"podcastPalette\":{\"Vibrant\":{\"rgb\":[196,172,132],\"population\":972},\"DarkVibrant\":{\"rgb\":[132,101,60],\"population\":585},\"LightVibrant\":{\"rgb\":[220,204,181],\"population\":5},\"Muted\":{\"rgb\":[156,124,84],\"population\":3},\"DarkMuted\":{\"rgb\":[143,115,77],\"population\":34},\"LightMuted\":{\"rgb\":[204,193,176],\"population\":246}},\"pageThemes\":{\"podcast\":null},\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#8d493a\",\"primary\":{\"r\":141,\"g\":73,\"b\":58,\"a\":1},\"primary_hover\":{\"r\":122,\"g\":56,\"b\":43,\"a\":1},\"primary_elevated\":{\"r\":122,\"g\":56,\"b\":43,\"a\":1},\"secondary\":{\"r\":141,\"g\":73,\"b\":58,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":141,\"g\":73,\"b\":58,\"a\":0.2},\"bg_hover\":{\"r\":141,\"g\":73,\"b\":58,\"a\":0.3},\"dark\":{\"primary\":{\"r\":141,\"g\":73,\"b\":58,\"a\":1},\"primary_hover\":{\"r\":161,\"g\":90,\"b\":74,\"a\":1},\"primary_elevated\":{\"r\":161,\"g\":90,\"b\":74,\"a\":1},\"secondary\":{\"r\":141,\"g\":73,\"b\":58,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":141,\"g\":73,\"b\":58,\"a\":0.2},\"bg_hover\":{\"r\":141,\"g\":73,\"b\":58,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":141,\"g\":73,\"b\":58,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":192,\"g\":118,\"b\":100,\"a\":1}}},\"bg\":{\"name\":\"#bd6800\",\"hue\":{\"r\":189,\"g\":104,\"b\":0,\"a\":1},\"tint\":{\"r\":189,\"g\":104,\"b\":0,\"a\":0.1},\"primary\":{\"r\":248.4,\"g\":239.9,\"b\":229.5,\"a\":1},\"primary_hover\":{\"r\":243.89999999999998,\"g\":235.39999999999998,\"b\":225,\"a\":1},\"primary_elevated\":{\"r\":243.89999999999998,\"g\":235.39999999999998,\"b\":225,\"a\":1},\"secondary\":{\"r\":233.10000000000002,\"g\":224.60000000000002,\"b\":214.2,\"a\":1},\"secondary_elevated\":{\"r\":203.3020442353792,\"g\":195.0232990149694,\"b\":184.88721582340014,\"a\":1},\"tertiary\":{\"r\":216,\"g\":207.5,\"b\":197.1,\"a\":1},\"quaternary\":{\"r\":182.70000000000002,\"g\":174.2,\"b\":163.79999999999998,\"a\":1},\"dark\":{\"primary\":{\"r\":38.7,\"g\":31.099999999999998,\"b\":21.599999999999998,\"a\":1},\"primary_hover\":{\"r\":43.2,\"g\":35.6,\"b\":26.1,\"a\":1},\"primary_elevated\":{\"r\":43.2,\"g\":35.6,\"b\":26.1,\"a\":1},\"secondary\":{\"r\":50.400000000000006,\"g\":43.7,\"b\":33.300000000000004,\"a\":1},\"secondary_elevated\":{\"r\":57.97847946519652,\"g\":51.12536699385489,\"b\":40.51800235490517,\"a\":1},\"tertiary\":{\"r\":67.5,\"g\":59.900000000000006,\"b\":49.5,\"a\":1},\"quaternary\":{\"r\":99.9,\"g\":92.3,\"b\":81.89999999999999,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#8d493a\",\"primary\":{\"r\":141,\"g\":73,\"b\":58,\"a\":1},\"primary_hover\":{\"r\":123,\"g\":64,\"b\":51,\"a\":1},\"primary_elevated\":{\"r\":141,\"g\":73,\"b\":58,\"a\":1},\"secondary\":{\"r\":141,\"g\":73,\"b\":58,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":141,\"g\":73,\"b\":58,\"a\":1},\"primary_hover\":{\"r\":161,\"g\":90,\"b\":74,\"a\":1},\"primary_elevated\":{\"r\":161,\"g\":90,\"b\":74,\"a\":1},\"secondary\":{\"r\":141,\"g\":73,\"b\":58,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":141,\"g\":73,\"b\":58,\"a\":0.2},\"bg_hover\":{\"r\":141,\"g\":73,\"b\":58,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":84,\"g\":72,\"b\":61,\"a\":1},\"secondary\":{\"r\":132,\"g\":128,\"b\":124,\"a\":1},\"tertiary\":{\"r\":142,\"g\":136,\"b\":130,\"a\":1},\"accent\":{\"r\":141,\"g\":73,\"b\":58,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":192,\"g\":118,\"b\":100,\"a\":1}}},\"bg\":{\"name\":\"#f8ede3\",\"hue\":{\"r\":248,\"g\":237,\"b\":227,\"a\":1},\"tint\":{\"r\":248,\"g\":237,\"b\":227,\"a\":1},\"primary\":{\"r\":248,\"g\":237,\"b\":227,\"a\":1},\"primary_hover\":{\"r\":233,\"g\":223,\"b\":213,\"a\":1},\"primary_elevated\":{\"r\":248,\"g\":237,\"b\":227,\"a\":1},\"secondary\":{\"r\":233,\"g\":223,\"b\":213,\"a\":1},\"secondary_elevated\":{\"r\":233,\"g\":223,\"b\":213,\"a\":1},\"tertiary\":{\"r\":214,\"g\":205,\"b\":196,\"a\":1},\"quaternary\":{\"r\":178,\"g\":170,\"b\":163,\"a\":1},\"dark\":{\"primary\":{\"r\":38.7,\"g\":31.099999999999998,\"b\":21.599999999999998,\"a\":1},\"primary_hover\":{\"r\":43.2,\"g\":35.6,\"b\":26.1,\"a\":1},\"primary_elevated\":{\"r\":43.2,\"g\":35.6,\"b\":26.1,\"a\":1},\"secondary\":{\"r\":50.400000000000006,\"g\":43.7,\"b\":33.300000000000004,\"a\":1},\"secondary_elevated\":{\"r\":57.97847946519652,\"g\":51.12536699385489,\"b\":40.51800235490517,\"a\":1},\"tertiary\":{\"r\":67.5,\"g\":59.900000000000006,\"b\":49.5,\"a\":1},\"quaternary\":{\"r\":99.9,\"g\":92.3,\"b\":81.89999999999999,\"a\":1}}},\"wordmark_bg\":{\"r\":248,\"g\":237,\"b\":227,\"a\":1},\"is_dark\":false}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[196,172,132],\"population\":825},\"DarkVibrant\":{\"rgb\":[132,101,60],\"population\":449},\"LightVibrant\":{\"rgb\":[212,188,156],\"population\":1},\"Muted\":{\"rgb\":[164,132,92],\"population\":9},\"DarkMuted\":{\"rgb\":[140,108,68],\"population\":38},\"LightMuted\":{\"rgb\":[207,194,176],\"population\":92}},\"section\":{\"id\":206570,\"created_at\":\"2025-04-02T23:25:22.621Z\",\"updated_at\":\"2026-04-24T13:10:21.528Z\",\"publication_id\":2874913,\"name\":\"Concepts\",\"description\":\"Learn one concept with your coffee.\",\"slug\":\"concepts\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":0,\"port_status\":\"success\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/d2946fd2-0953-4810-ad71-4d4cd4769c9f_1280x1280.png\",\"hide_from_navbar\":false,\"email_from_name\":\"The Coder Cafe\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"showLinks\":[],\"unifiedPodcastSettings\":null,\"pageTheme\":{\"id\":63961,\"publication_id\":2874913,\"section_id\":206570,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"left\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":false,\"customDomain\":\"read.thecoder.cafe\"},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":false,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":false,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":null,\"twitterPixelId\":null,\"parselyPixelId\":null,\"googleTagManagerToken\":\"GTM-KDC62RGC\",\"googleAnalytics4Token\":\"G-M9NDKF5ZJX\",\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":false,\"credit_token_enabled\":false,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":false,\"enable_drop_caps\":false,\"enable_post_page_conversion\":false,\"enable_prev_next_nav\":true,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":\"G-M9NDKF5ZJX\",\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"AI is getting better every day. Are you? At The Coder Cafe, we serve fundamental concepts to make you an engineer that AI won't replace. Written by a Google SWE, trusted by thousands of engineers worldwide.\",\"notes_feed_enabled\":true,\"paywall_unlock_tokens\":false,\"post_preview_crop_gravity\":\"center\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":false,\"reader_referrals_leaderboard_enabled\":false,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":false,\"local_currency_modal_seen\":true,\"local_payment_methods_modal_seen\":false,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":true,\"welcome_page_opt_out_text\":\"Let me read first.\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":false,\"holiday_gifting_post_header\":false,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://read.thecoder.cafe/p/metastable-failures\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":194114637,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2026-05-27T13:02:50.954Z\",\"updated_at\":\"2026-05-27T13:07:23.467Z\",\"publication_id\":2874913,\"search_engine_description\":\"Why some outages keep going long after the trigger is gone. A guide to metastable failures, feedback loops, and hidden capacity in distributed systems.\",\"search_engine_title\":null,\"section_id\":206570,\"should_send_free_preview\":false,\"show_guest_bios\":true,\"slug\":\"metastable-failures\",\"social_title\":\"Metastable Failures Explained: Why Fixing the Trigger Fails\",\"subtitle\":\"Why Fixing the Trigger Fails\",\"teaser_post_eligible\":true,\"title\":\"Metastable Failures in Distributed Systems\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"everyone\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":1,\"reactions\":{\"\u2764\":12},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"lsm-trees\",\"next_post_slug\":null,\"cover_image\":\"https://substackcdn.com/image/fetch/$s_!Su99!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png\",\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":{\"post_id\":194114637,\"podcast_episode_number\":null,\"podcast_season_number\":null,\"podcast_episode_type\":null,\"should_syndicate_to_other_feed\":null,\"syndicate_to_section_id\":null,\"hide_from_feed\":false,\"free_podcast_url\":null,\"free_podcast_duration\":null,\"preview_contains_ad\":false,\"was_imported_self_serve_sync\":false,\"draft_free_podcast_url\":null,\"draft_free_podcast_duration\":null},\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"The system is technically up, effectively down.\",\"body_html\":\"<p>\u2615 <em>Welcome to The Coder Cafe! Today, we explore one of the nastiest failure patterns in distributed systems: metastable failures. Based on the <a href=\\\"https://sigops.org/s/conferences/hotos/2021/papers/hotos21-s11-bronson.pdf\\\">Metastable Failures in Distributed Systems</a> whitepaper, we break down why these failures happen, why they persist, what we can do about them, and why our instinct to fix them is probably wrong. Get cozy, grab a coffee, and let\u2019s begin!</em></p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!Su99!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!Su99!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 424w, https://substackcdn.com/image/fetch/$s_!Su99!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 848w, https://substackcdn.com/image/fetch/$s_!Su99!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 1272w, https://substackcdn.com/image/fetch/$s_!Su99!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!Su99!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png\\\" width=\\\"1456\\\" height=\\\"728\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/e51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":728,"width":1456,"resizeWidth":null,"bytes":1646402,"alt":"The system is technically up, effectively down.","title":null,"type":"image/png","href":null,"belowTheFold":false,"topImage":true,"internalRedirect":"https://read.thecoder.cafe/i/194114637?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png","isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"The system is technically up, effectively down.\\\" title=\\\"The system is technically up, effectively down.\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!Su99!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 424w, https://substackcdn.com/image/fetch/$s_!Su99!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 848w, https://substackcdn.com/image/fetch/$s_!Su99!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 1272w, https://substackcdn.com/image/fetch/$s_!Su99!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe51a6988-d738-4c8c-a46e-576643bdf520_1600x800.png 1456w\\\" sizes=\\\"100vw\\\" fetchpriority=\\\"high\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><h1>Stable, Vulnerable, Metastable</h1><p>Metastable failures borrow their name from physics, where <strong>metastable means something that looks stable but isn\u2019t</strong>. To understand how a distributed system can end up in such a state, we need to look at three distinct states it can be in:</p><ul><li><p><strong>Stable:</strong> The system recovers on its own after any disruption. This is what we call resilience in <em><a href=\\\"https://read.thecoder.cafe/p/resilient-fault-tolerant-robust-reliable\\\">Resilient, Fault-tolerant, Robust, or Reliable</a></em>.</p></li><li><p><strong>Vulnerable</strong>: The system looks perfectly healthy, but it's operating above its <strong>hidden capacity</strong>: the load level below which it can self-heal from any disruption. It responds fast, metrics are green, and nothing is alarming. Many production systems deliberately operate here because it's more efficient: resources are used closer to their limit. But there's <strong>no slack left</strong>. And the deeper the system operates in a vulnerable state, the smaller the trigger needed to push it over the edge. Indeed, a system just above its hidden capacity can survive large disruptions; a system near its advertised capacity can be tipped by almost anything.</p></li><li><p><strong>Metastable failure</strong>: A trigger (e.g., a network blip, a deployment, a traffic spike) pushes the system over its hidden capacity. The system is not fully broken: processes are alive, and it\u2019s still running. But goodput collapses: it\u2019s no longer doing any useful work. <strong>Technically up, effectively down</strong>. And unlike a regular outage, removing the trigger doesn\u2019t fix it. Getting out requires a strong corrective push: a restart, a dramatic load reduction, a manual intervention.</p><blockquote><p><strong>NOTE</strong>: <em>If you\u2019re not familiar with the concept of goodput, it\u2019s the throughput of useful work completed successfully. For example, in a web application receiving 1000 requests per second but returning errors for 800 of them, the goodput is only 200 RPS.</em></p></blockquote></li></ul><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!qhkv!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!qhkv!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 424w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 848w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 1272w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!qhkv!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png\\\" width=\\\"1456\\\" height=\\\"320\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/a54abb43-fd09-4675-bb91-b695db389435_1640x360.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":320,"width":1456,"resizeWidth":null,"bytes":56976,"alt":"Diagram showing the three states of a system experiencing a metastable failure: Stable, Vulnerable, and Metastable. Arrows show transitions: increasing load moves the system from Stable to Vulnerable, a trigger moves it from Vulnerable to Metastable, and a strong corrective action (dashed arrow) brings it back to Stable.","title":null,"type":"image/png","href":null,"belowTheFold":false,"topImage":false,"internalRedirect":"https://read.thecoder.cafe/i/194114637?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png","isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"Diagram showing the three states of a system experiencing a metastable failure: Stable, Vulnerable, and Metastable. Arrows show transitions: increasing load moves the system from Stable to Vulnerable, a trigger moves it from Vulnerable to Metastable, and a strong corrective action (dashed arrow) brings it back to Stable.\\\" title=\\\"Diagram showing the three states of a system experiencing a metastable failure: Stable, Vulnerable, and Metastable. Arrows show transitions: increasing load moves the system from Stable to Vulnerable, a trigger moves it from Vulnerable to Metastable, and a strong corrective action (dashed arrow) brings it back to Stable.\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!qhkv!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 424w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 848w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 1272w, https://substackcdn.com/image/fetch/$s_!qhkv!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa54abb43-fd09-4675-bb91-b695db389435_1640x360.png 1456w\\\" sizes=\\\"100vw\\\"></picture><div></div></div></a><figcaption class=\\\"image-caption\\\">The three states of a metastable failure. A system can drift into the vulnerable state unnoticed, and a single trigger is enough to push it into the metastable state it cannot escape on its own.</figcaption></figure></div><p>The most disorienting property of a metastable failure: stopping the trigger doesn\u2019t stop the failure. To understand why, we need to talk about feedback loops.</p><h1>The Feedback Loop is The Root Cause</h1><p>In a previous post on <a href=\\\"https://read.thecoder.cafe/p/systems-thinking\\\">Systems Thinking Explained</a>, we defined a feedback loop as: If <code>A</code> causes <code>B</code>, then <code>B</code> influences <code>A</code>.</p><p>A feedback loop is exactly the mechanism that keeps a system <strong>stuck in the metastable state</strong>. There is always a sustaining effect, a feedback loop, that prevents recovery. The trigger is just what pushes the system over the edge. The loop is what keeps it there. Blaming the trigger is the natural instinct, and almost always the wrong diagnosis.</p><p>Let\u2019s discuss a concrete example to make this clear.</p><p>Imagine a web application that queries a database. The database comfortably handles up to 300 QPS. The application retries any query that doesn\u2019t respond within 1 second. The system is running at 280 QPS, healthy and fast, within the database\u2019s capacity.</p><p>Then, a transient network issue occurs for 10 seconds. When the issue is over, all the queued requests flood in at once. The database gets hit with a surge it can\u2019t absorb: latency spikes and queries start timing out. So the application retries them. This doubles the effective load to 560 QPS. The database, already struggling, falls further behind. More timeouts. More retries. The loop is now self-sustaining:</p><blockquote><p><em>High load \u2192 Timeouts \u2192 Retries \u2192 Higher load \u2192 More timeouts \u2192 More retries</em></p></blockquote><p>The transient network issue was fixed minutes ago. Yet, the system is still completely broken. The trigger is gone; <strong>the feedback loop is not</strong>. The only way out is to dramatically cut the load or disable retries entirely.</p><p>This is a <strong>metastable failure</strong>. The system was vulnerable because it was operating close to its <strong>hidden capacity</strong>. A minor, transient <strong>trigger</strong> pushed it over the edge and into a self-sustaining failure state it couldn\u2019t escape on its own. The retry mechanism, a feature designed to improve reliability, became the very thing that prevented recovery.</p><p>This is one example, but the same pattern appears with caches, connection pools, failover logic, and more. The shape is always the same: <strong>a feedback loop that turns a temporary problem into a permanent one</strong>.</p><h1>Why It\u2019s So Hard To Fix</h1><p>Two things make metastable failures particularly nasty.</p><ul><li><p><strong>We can be tempted to blame the wrong thing</strong>. When an outage happens, the trigger is what\u2019s visible and recent: a spike, a deployment, a hardware fault. It\u2019s the obvious culprit. But the trigger only exposed the problem; it didn\u2019t create it. The sustaining feedback loop was already there, structural and invisible. When analyzing the problem in retrospect, teams focus on the trigger; fixes address the trigger; and the system remains vulnerable to the next one. The authors of the paper observed teams declare a metastable failure \u201Cresolved\u201D multiple times before realizing the real cause had never been touched.</p></li><li><p><strong>The feedback loop grows stronger with scale</strong>. Small-scale tests won\u2019t reveal it. A staging environment running at 10% capacity may handle the same trigger without falling into a metastable state, because the loop isn\u2019t strong enough at that scale to be self-sustaining. This means these failures can slip past even rigorous testing regimes and only manifest in production at full load.</p></li></ul><p>We defined hidden capacity earlier as the load level below which the system can self-heal from any disruption. It\u2019s different, and always lower, than the advertised capacity. In our example, the numbers make it concrete: the advertised capacity is 300 QPS, but the hidden capacity is only 150 QPS, because retries double the load under failure. The gap between those two numbers is <strong>where vulnerability lives</strong>.</p><p>Measuring the hidden capacity is not straightforward, though. One possible approach is to apply a trigger at a given load level and observe whether the system recovers on its own:</p><ul><li><p>If it does, we are below the hidden capacity.</p></li><li><p>If it doesn\u2019t, we are above it.</p></li></ul><p>We can also estimate it indirectly: in the retry example, retries double the load under failure, so the hidden capacity is roughly half the advertised capacity.</p><h1>Finding the Loop, Not the Bug</h1><p><strong>Metastable failures are not bugs</strong>. We can\u2019t write a unit test that catches them. They are emergent behaviors: properties that arise from the interaction of a system\u2019s components under specific conditions, not logic errors in any individual component. No single piece of code is buggy, no single configuration is wrong. The failure is a consequence of how everything fits together under load.</p><p>This changes how we need to think about them. The right question after an outage is not \u201C<em>What failed?</em>\u201D but \u201C<em>What loop sustained it?</em>\u201D And before an outage, the danger is not having bugs; it\u2019s optimizing so aggressively for efficiency that <strong>we push the system deeper into the vulnerable state without realizing it</strong>.</p><p>Retries, caches, failover logic, connection pools: these are all features that improve reliability in the common case. They are also, under the right conditions, the sustaining mechanisms of metastable failures. The same design decision that makes a system more resilient in normal operation can also prevent it from recovering when things go wrong.</p><h1>What Can We Do About It?</h1><p>The paper describes several approaches to reduce the risk of metastable failures:</p><ul><li><p><strong>Retry budgets and circuit breakers</strong>: Instead of retrying indefinitely, cap the total number of retries in flight at any given time. This directly weakens the feedback loop by limiting work amplification.</p></li><li><p><strong>LIFO scheduling under overload</strong>: Counterintuitively, switching from FIFO to LIFO when the system is overloaded allows some requests to complete within their deadline, preserving goodput instead of letting every request time out.</p><blockquote><p><strong>NOTE</strong>: <em>I already wrote a post about that approach in <a href=\\\"https://read.thecoder.cafe/p/adaptive-lifo\\\">Adaptive LIFO</a>.</em></p></blockquote></li><li><p><strong>Fast error paths</strong>: Success paths are heavily optimized, but error paths often aren\u2019t. An expensive error path (stack traces, DNS lookups, disk writes) under high failure rates can itself become a sustaining mechanism. Optimizing error paths reduces this risk.</p></li><li><p><strong>Read-through caches over look-aside caches</strong>: A read-through cache (where the cache itself fetches missing data from the database) can continue filling itself even when the application has given up on a request, steadily increasing the hit rate and helping the system recover. A look-aside cache (where the application is responsible for populating the cache) can\u2019t.</p></li><li><p><strong>Production stress testing</strong>: Small-scale tests won\u2019t reveal metastable failures. Testing against a portion of production traffic, with engineers ready to intervene, is the most reliable way to surface them.</p></li></ul><p>A note of humility from the paper: there is no systematic solution yet. These are ad-hoc mitigations developed in response to known failures. Detecting vulnerable states before they collapse remains an open problem.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://read.thecoder.cafe\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!ZoDz!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 424w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 848w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 1272w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!ZoDz!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png\\\" width=\\\"449\\\" height=\\\"224.5\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/c5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":600,"width":1200,"resizeWidth":449,"bytes":82853,"alt":"The Coder Cafe: Learn One Concept With Your Coffee.","title":"The Coder Cafe: Learn One Concept With Your Coffee.","type":"image/png","href":"https://read.thecoder.cafe","belowTheFold":true,"topImage":false,"internalRedirect":"https://www.thecoder.cafe/i/151119215?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png","isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"The Coder Cafe: Learn One Concept With Your Coffee.\\\" title=\\\"The Coder Cafe: Learn One Concept With Your Coffee.\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!ZoDz!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 424w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 848w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 1272w, https://substackcdn.com/image/fetch/$s_!ZoDz!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc5b47134-fe05-42e3-9aaf-dd2758923a98_1200x600.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><div class=\\\"subscription-widget-wrap-editor\\\" data-attrs=\\\"{"url":"https://read.thecoder.cafe/subscribe?","text":"Subscribe","language":"en"}\\\" data-component-name=\\\"SubscribeWidgetToDOM\\\"><div class=\\\"subscription-widget show-subscribe\\\"><div class=\\\"preamble\\\"><p class=\\\"cta-caption\\\">AI is getting better every day. Are you? At The Coder Cafe, we serve fundamental concepts to make you an engineer that AI won\u2019t replace. Written by a Google SWE, trusted by thousands of engineers worldwide.</p></div><form class=\\\"subscription-widget-subscribe\\\"><input type=\\\"email\\\" class=\\\"email-input\\\" name=\\\"email\\\" placeholder=\\\"Type your email\u2026\\\" tabindex=\\\"-1\\\"><input type=\\\"submit\\\" class=\\\"button primary\\\" value=\\\"Subscribe\\\"><div class=\\\"fake-input-wrapper\\\"><div class=\\\"fake-input\\\"></div><div class=\\\"fake-button\\\"></div></div></form></div></div><h1>Summary</h1><ul><li><p>A distributed system can pass through three states: stable, vulnerable, and metastable. The vulnerable state looks healthy, but it isn\u2019t.</p></li><li><p>The threshold between stable and vulnerable is invisible. Systems can operate in the vulnerable state for months without any sign of trouble.</p></li><li><p>When a trigger pushes a vulnerable system into a metastable failure, a feedback loop sustains the failure even after the trigger is gone.</p></li><li><p>The trigger is not the root cause. The feedback loop is. Fixing the trigger leaves the system vulnerable to the next one.</p></li><li><p>Reliability features like retries and caches can become the sustaining mechanism of a metastable failure under the right conditions.</p></li><li><p>Metastable failures are emergent behaviors, not bugs. We can\u2019t unit test for them, and optimizing for efficiency makes them more likely.</p></li><li><p>Mitigations exist (retry budgets, circuit breakers, LIFO scheduling, fast error paths), but they are all ad-hoc responses to known failures. Detecting vulnerable states before they collapse remains an open problem.</p></li></ul><h1>Resources</h1><h2>More From the <em><a href=\\\"https://read.thecoder.cafe/t/reliability\\\">Reliability</a></em> Category</h2><ul><li><p><a href=\\\"https://read.thecoder.cafe/p/resilient-fault-tolerant-robust-reliable\\\">Resilient, Fault-tolerant, Robust, or Reliable?</a></p></li><li><p><a href=\\\"https://read.thecoder.cafe/p/adaptive-lifo\\\">Adaptive LIFO</a></p></li><li><p><a href=\\\"https://read.thecoder.cafe/p/fail-open-fail-closed\\\">Fail Open vs. Fail Closed</a></p></li></ul><h2>Sources</h2><ul><li><p><a href=\\\"https://sigops.org/s/conferences/hotos/2021/papers/hotos21-s11-bronson.pdf\\\">Metastable Failures in Distributed Systems</a></p></li></ul><h2>Explore Further</h2><ul><li><p><a href=\\\"https://brooker.co.za/blog/2021/05/24/metastable.html\\\">Metastability and Distributed Systems</a></p></li></ul>\",\"has_dynamic_content\":false,\"truncated_body_text\":null,\"wordcount\":1687,\"post_preview_limit\":0,\"language\":\"en\",\"postTags\":[{\"id\":\"63ebdd74-f1c5-466c-80b0-33d8f7ac0e8e\",\"publication_id\":2874913,\"name\":\"Reliability\",\"slug\":\"reliability\",\"hidden\":false}],\"postCountryBlocks\":[],\"headlineTest\":null,\"coverImagePalette\":{\"Vibrant\":{\"rgb\":[63.75,191.25,148.74999999999997],\"population\":0},\"DarkVibrant\":{\"rgb\":[12,36,28],\"population\":383},\"LightVibrant\":{\"rgb\":[155.54999999999998,221.85,199.75],\"population\":0},\"Muted\":{\"rgb\":[116,132,124],\"population\":4},\"DarkMuted\":{\"rgb\":[44,66,60],\"population\":385},\"LightMuted\":{\"rgb\":[154,159,154],\"population\":206}},\"publishedBylines\":[{\"id\":3529974,\"name\":\"Teiva Harsanyi\",\"handle\":\"teivah\",\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/e800897a-277b-458c-a1a2-48b8b1b8d70c_500x500.png\",\"bio\":\"Software Engineer at Google, Author, Speaker.\",\"profile_set_up_at\":\"2022-01-29T15:48:31.516Z\",\"reader_installed_at\":\"2024-08-13T23:37:49.173Z\",\"publicationUsers\":[{\"id\":2922208,\"user_id\":3529974,\"publication_id\":2874913,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":2874913,\"name\":\"The Coder Cafe\",\"subdomain\":\"thecodercafe\",\"custom_domain\":\"read.thecoder.cafe\",\"custom_domain_optional\":false,\"hero_text\":\"AI is getting better every day. Are you? At The Coder Cafe, we serve fundamental concepts to make you an engineer that AI won't replace. Written by a Google SWE, trusted by thousands of engineers worldwide.\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/a08792c7-cadb-478c-bce4-a10c5dc5ac05_1280x1280.png\",\"author_id\":3529974,\"primary_user_id\":3529974,\"theme_var_background_pop\":\"#FF6719\",\"created_at\":\"2024-08-11T19:08:26.110Z\",\"email_from_name\":\"The Coder Cafe\",\"copyright\":\"Teiva Harsanyi\",\"founding_plan_name\":\"Founding Member\",\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"enabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"newspaper\",\"is_personal_mode\":false,\"logo_url_wide\":\"https://substack-post-media.s3.amazonaws.com/public/images/09e5dcc0-f9fe-47f0-a81e-0148d03f7535_1344x256.png\"}},{\"id\":2922199,\"user_id\":3529974,\"publication_id\":2874904,\"role\":\"admin\",\"public\":true,\"is_primary\":false,\"publication\":{\"id\":2874904,\"name\":\"Teiva Harsanyi\",\"subdomain\":\"teivah\",\"custom_domain\":null,\"custom_domain_optional\":false,\"hero_text\":\"\uD83D\uDCD6 100 Go Mistakes author (2022)\\nSoftware engineer in Go, Cloud Computing, Reliability, Observability, Performance\\n\",\"logo_url\":null,\"author_id\":3529974,\"primary_user_id\":null,\"theme_var_background_pop\":\"#FF6719\",\"created_at\":\"2024-08-11T19:06:14.237Z\",\"email_from_name\":null,\"copyright\":\"Teiva Harsanyi\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"profile\",\"is_personal_mode\":true,\"logo_url_wide\":null}}],\"is_guest\":false,\"bestseller_tier\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":1,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"subscriber\",\"tier\":1,\"accent_colors\":null},\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":12,\"comment_count\":2,\"child_comment_count\":1,\"audio_items\":[{\"post_id\":194114637,\"voice_id\":\"en-US-OnyxTurboMultilingualNeural\",\"audio_url\":\"https://substack-video.s3.amazonaws.com/video_upload/post/194114637/tts/cffbea4a-bd3b-474c-9925-d0035c02da16/en-US-OnyxTurboMultilingualNeural.mp3\",\"type\":\"tts\",\"status\":\"completed\"}],\"is_geoblocked\":false,\"hasCashtag\":false,\"section\":{\"id\":206570,\"created_at\":\"2025-04-02T23:25:22.621Z\",\"updated_at\":\"2026-04-24T13:10:21.528Z\",\"publication_id\":2874913,\"name\":\"Concepts\",\"description\":\"Learn one concept with your coffee.\",\"slug\":\"concepts\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":0,\"port_status\":\"success\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/d2946fd2-0953-4810-ad71-4d4cd4769c9f_1280x1280.png\",\"hide_from_navbar\":false,\"email_from_name\":\"The Coder Cafe\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"showLinks\":[],\"unifiedPodcastSettings\":null,\"pageTheme\":{\"id\":63961,\"publication_id\":2874913,\"section_id\":206570,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"left\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}},\"comments\":[{\"id\":266416976,\"body\":\"That was a great read\u2026and I realised something.\\n\\nThe individual human system (biologically) is a one of the largest distributed system. If there is a \u201Cmetastable failure\u201D in this system, there is a disease. Sometimes the human system recovers\u2026but sometimes it cant. (Note that I am not talking about the natural lifespan ending but chronic or external diseases/issues).\\n\\nHow has this system bettered over time? Through evolution. Of course, now, the system is so complex that it is generally adaptive even without evolution.\\n\\nKeeping that in mind, Evolutionary/Genetic algorithms, Generative Adversarial Networks, and Reinforcement learning seem to be some things which could work for this use case.\",\"body_json\":{\"type\":\"doc\",\"attrs\":{\"schemaVersion\":\"v1\",\"title\":null},\"content\":[{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"That was a great read\u2026and I realised something.\"}]},{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"The individual human system (biologically) is a one of the largest distributed system. If there is a \u201Cmetastable failure\u201D in this system, there is a disease. Sometimes the human system recovers\u2026but sometimes it cant. (Note that I am not talking about the natural lifespan ending but chronic or external diseases/issues).\"}]},{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"How has this system bettered over time? Through evolution. Of course, now, the system is so complex that it is generally adaptive even without evolution.\"}]},{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"Keeping that in mind, Evolutionary/Genetic algorithms, Generative Adversarial Networks, and Reinforcement learning seem to be some things which could work for this use case.\"}]}]},\"publication_id\":2874913,\"post_id\":194114637,\"user_id\":508383034,\"ancestor_path\":\"\",\"type\":\"comment\",\"deleted\":false,\"date\":\"2026-05-28T06:55:15.750Z\",\"edited_at\":\"2026-05-28T06:55:38.776Z\",\"status\":\"published\",\"pinned_by_user_id\":null,\"restacks\":0,\"name\":\"Tushar Raturi\",\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/c2027700-2e2c-410b-bc1a-3b249f88ed17_320x322.png\",\"handle\":\"tusharraturi\",\"reactor_names\":[\"Teiva Harsanyi\"],\"reaction\":null,\"reactions\":{\"\u2764\":1},\"reaction_count\":1,\"children\":[],\"bans\":[],\"suppressed\":false,\"user_banned\":false,\"user_banned_for_comment\":false,\"user_slug\":\"tusharraturi\",\"metadata\":{\"is_author\":false,\"membership_state\":\"free_signup\",\"eligibleForGift\":true,\"author_on_other_pub\":{\"name\":\"Binary Tantra\",\"id\":9041176,\"base_url\":\"https://binarytantra.substack.com\"}},\"user_bestseller_tier\":null,\"can_dm\":true,\"userStatus\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null},\"score\":6,\"children_count\":1,\"reported_by_user\":false,\"restacked\":false,\"childrenSummary\":\"1 reply by Teiva Harsanyi\"}],\"canonicalUrl\":\"https://read.thecoder.cafe/p/metastable-failures\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://read.thecoder.cafe/p/metastable-failures\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":\"#8d493a\",\"background_pop\":\"#8d493a\",\"color_theme_bg_web\":\"#f8ede3\",\"cover_bg_color\":\"#f8ede3\",\"cover_bg_color_secondary\":\"#e9dfd5\",\"background_pop_darken\":\"#7b4033\",\"print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_darken\":\"#7b4033\",\"color_theme_print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_20\":\"rgba(141, 73, 58, 0.2)\",\"color_theme_bg_pop_30\":\"rgba(141, 73, 58, 0.3)\",\"print_pop\":\"#8d493a\",\"color_theme_accent\":\"#8d493a\",\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#8d493a\",\"home_hero\":\"newspaper\",\"home_posts\":\"custom\",\"home_show_top_posts\":true,\"web_bg_color\":\"#f8ede3\",\"background_contrast_1\":\"#e9dfd5\",\"background_contrast_2\":\"#d6cdc4\",\"background_contrast_3\":\"#b2aaa3\",\"background_contrast_4\":\"#8e8882\",\"background_contrast_5\":\"#4e4b48\",\"color_theme_bg_contrast_1\":\"#e9dfd5\",\"color_theme_bg_contrast_2\":\"#d6cdc4\",\"color_theme_bg_contrast_3\":\"#b2aaa3\",\"color_theme_bg_contrast_4\":\"#8e8882\",\"color_theme_bg_contrast_5\":\"#4e4b48\",\"color_theme_bg_elevated\":\"#f8ede3\",\"color_theme_bg_elevated_secondary\":\"#e9dfd5\",\"color_theme_bg_elevated_tertiary\":\"#d6cdc4\",\"color_theme_detail\":\"#dfd5cc\",\"background_contrast_pop\":\"rgba(141, 73, 58, 0.4)\",\"color_theme_bg_contrast_pop\":\"rgba(141, 73, 58, 0.4)\",\"theme_bg_is_dark\":\"0\",\"print_on_web_bg_color\":\"hsl(28.571428571428594, 15.688073394495422%, 28.54901960784314%)\",\"print_secondary_on_web_bg_color\":\"#84807c\",\"background_pop_rgb\":\"141, 73, 58\",\"color_theme_bg_pop_rgb\":\"141, 73, 58\",\"color_theme_accent_rgb\":\"141, 73, 58\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":null,\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"125.0\",\"major\":\"125\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
|
||
<script nonce="SiMLZxbT83ypi3YnctfvQQ">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"thecodercafe\",\"publication_id\":2874913,\"has_plans\":true,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"section_name\":\"Concepts\",\"section_slug\":\"concepts\",\"section_id\":206570,\"section_is_podcast\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
|
||
|
||
|
||
|
||
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.7371871e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.571db0c6.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.a0731d37.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.d196682b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.2686bb5d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.ae8857e4.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.4614ce58.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.41564edd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.b2140fcd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.f3dc2b43.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.7aa24e85.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.1820773d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.765aca5a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.2f1bdde5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.4e726ad5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.f03f24de.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.f579d62b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.80c75c78.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.92b9fc9c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.258e49f2.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.d493457b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.ce61e623.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.015b0f71.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.f1207089.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.b91ab2ef.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.b93f6845.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89356.66161d1e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.211a7337.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47925.24a01d36.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.03bd4c41.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71505.3dc43e33.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.5d148f2c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.86d95e8c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.a5a682c3.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/86118.ff1ec843.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.7fe128ac.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.c78aabc1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.d6e9f8ec.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/29231.a7af5911.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.2b32bbb6.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.f1c17ded.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.1d0af2b9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.951c11cb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.58e078b0.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.dfde36c8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.94c6b3f1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.a327e9f4.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.8bb0d59c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.8f5c5991.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.cb326b80.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.680833b5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.df6a856d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.cf3bcc2c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.8a1bc5dd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.3f20fb36.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.bdbc4e21.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58483.3adddb8a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.0a97506f.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.09a9082f.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.7f0face7.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/92610.a4fc1daf.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.cb10b4b8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11244.b723f409.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99260.28b0ed6a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.7d5c5ba3.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/76768.9a2f2bae.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/41932.6c688f22.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12114.979fb38e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.9732c6f9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.4f5ae3be.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53871.2923c5c9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.588a06d1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99068.bd2e1fba.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.79d3f415.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.c7f428fb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.098bcbcf.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.2eb51db5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.72d1d844.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.bbf1c283.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.2e8ed950.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/32552.889dd584.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.c287f2d8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.6c265246.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.9b9761ae.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.b40bf629.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.07cd629b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.35b48b09.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.f37fb930.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.24075434.js" charset="utf-8"></script>
|
||
|
||
|
||
<script nonce="SiMLZxbT83ypi3YnctfvQQ" nomodule>
|
||
(function() {
|
||
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
|
||
var warningDiv = document.createElement('div');
|
||
warningDiv.style.color = 'red';
|
||
warningDiv.style.padding = '10px';
|
||
warningDiv.style.margin = '10px 0';
|
||
warningDiv.style.border = '1px solid red';
|
||
warningDiv.style.backgroundColor = 'lightyellow';
|
||
warningDiv.innerText = message;
|
||
document.body.prepend(warningDiv);
|
||
})();
|
||
</script>
|
||
|
||
|
||
<!-- Datadog Analytics -->
|
||
<script nonce="SiMLZxbT83ypi3YnctfvQQ">
|
||
(function(h,o,u,n,d) {
|
||
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
|
||
d=o.createElement(u);d.async=1;d.src=n
|
||
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
|
||
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
|
||
window.DD_RUM.onReady(function() {
|
||
window.DD_RUM.init({
|
||
clientToken: 'puba71073f072643721169b68f352438710',
|
||
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
|
||
site: 'datadoghq.com',
|
||
service: 'web',
|
||
env: window._preloads.dd_env,
|
||
version: '8b4c00716b2bc0f7ca1fded553014f7e37b5b07a',
|
||
sessionSampleRate: 1,
|
||
sessionReplaySampleRate: 10,
|
||
trackUserInteractions: window._preloads.dd_ti,
|
||
trackResources: true,
|
||
trackLongTasks: true,
|
||
defaultPrivacyLevel: 'mask-user-input',
|
||
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
|
||
});
|
||
})
|
||
</script>
|
||
<!-- End Datadog Analytics -->
|
||
|
||
<!-- Cloudflare Web Analytics -->
|
||
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
|
||
<!-- End Cloudflare Web Analytics -->
|
||
|
||
|
||
<!-- Fallback tracking pixels -->
|
||
|
||
|
||
|
||
|
||
<noscript>
|
||
<style>
|
||
#nojs-banner {
|
||
position: fixed;
|
||
bottom: 0;
|
||
left: 0;
|
||
padding: 16px 16px 16px 32px;
|
||
width: 100%;
|
||
box-sizing: border-box;
|
||
background: red;
|
||
color: white;
|
||
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
|
||
font-size: 13px;
|
||
line-height: 13px;
|
||
}
|
||
#nojs-banner a {
|
||
color: inherit;
|
||
text-decoration: underline;
|
||
}
|
||
</style>
|
||
|
||
<div id="nojs-banner">
|
||
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
|
||
</div>
|
||
</noscript>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="SiMLZxbT83ypi3YnctfvQQ" data-cf-beacon='{"version":"2024.11.0","token":"68cfe66b5c4749e2ba64d4d9640c04c0","spa":2}' crossorigin="anonymous"></script>
|
||
</body>
|
||
</html>
|