Files
nexus/sreweekly/articles/536/07-rollback-does-not-erase-distributed-memory.html

343 lines
133 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8" />
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
<link rel="preconnect" href="https://substackcdn.com" />
<title data-rh="true">Rollback Does Not Erase Distributed Memory</title>
<meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Rollback Does Not Erase Distributed Memory"/><meta data-rh="true" name="twitter:title" content="Rollback Does Not Erase Distributed Memory"/><meta data-rh="true" name="description" content="The rollback finished successfully."/><meta data-rh="true" property="og:description" content="The rollback finished successfully."/><meta data-rh="true" name="twitter:description" content="The rollback finished successfully."/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!gHJA!,w_1200,h_675,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!tEEV!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fstoriesfromtheedge.substack.com%2Fapi%2Fv1%2Fpost_preview%2F216113519%2Ftwitter.jpg%3Fversion%3D4"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" property="article:modified_time" content="2026-09-18T00:45:41.822Z"/>
<style>
@layer legacy, pencraftReset, pencraft;
</style>
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.91a878a369a8d9c64226.css" />
<link rel="preload" as="font" href="https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M5knjsS_ul.woff2" crossorigin />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/2732.405afd64.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/8511.356c16b0.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/4806.80f000cc.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/195.090c9fca.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/8637.df7b0eae.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/6204.d38297b3.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/1133.61d565c6.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/6108.0847f260.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/2828.bba18efc.css" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
<meta name="author" content="balu Kambala" />
<meta property="og:url" content="https://storiesfromtheedge.substack.com/p/rollback-does-not-erase-distributed" />
<link rel="canonical" href="https://storiesfromtheedge.substack.com/p/rollback-does-not-erase-distributed" />
<link rel="shortcut icon" href="https://substackcdn.com/icons/substack/favicon.ico" sizes="32x32">
<link rel="icon" href="https://substackcdn.com/icons/substack/icon.svg" type="image/svg+xml">
<link rel="apple-touch-icon" href="https://substackcdn.com/icons/substack/apple-touch-icon.png">
<link rel="alternate" type="application/rss+xml" href="/feed" title="stories from the edge"/>
<style>
@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8on7mTNmnUHowCw.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onXmTNmnUHowCw.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onTmTNmnUHowCw.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onrmTNmnUHo.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M9knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M2knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M3knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M5knjsS_ul.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3FafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3OafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3PafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3BafaPWnII.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
</style>
<style data-pub-theme-prefix>:root{--color_theme_bg_pop:#FF6719;--background_pop:#FF6719;--cover_bg_color:#FFFFFF;--cover_bg_color_secondary:#f0f0f0;--background_pop_darken:#ff5600;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#ff5600;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(255, 103, 25, 0.2);--color_theme_bg_pop_30:rgba(255, 103, 25, 0.3);--print_pop:#ff6719;--color_theme_accent:#ff6719;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#ff6719;--home_hero:newspaper;--home_posts:list;--background_contrast_1:#f0f0f0;--background_contrast_2:#dddddd;--background_contrast_3:#b7b7b7;--background_contrast_4:#929292;--background_contrast_5:#515151;--color_theme_detail:#e6e6e6;--background_contrast_pop:rgba(255, 103, 25, 0.4);--color_theme_bg_contrast_pop:rgba(255, 103, 25, 0.4);--theme_bg_is_dark:0;--background_pop_rgb:255, 103, 25;--color_theme_bg_pop_rgb:255, 103, 25;--color_theme_accent_rgb:255, 103, 25;}</style>
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.91a878a369a8d9c64226.css" />
<style data-pub-theme-postfix></style>
</head>
<body class="">
<div id="entry">
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);--publication-post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:71px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="flex-basis:0px;flex-grow:1;" class="logoContainer-p12gJb"><a data-native href="/" class="pencraft pc-display-contents pc-reset"><div draggable="false" class="pencraft pc-display-flex pc-position-relative pc-reset"><div style="width:40px;height:40px;" class="pencraft pc-display-flex pc-reset bg-white-ZBV5av pc-borderRadius-sm overflow-hidden-WdpwT6 sizing-border-box-DggLA4"><picture class="pencraft pc-display-contents pc-reset"><source type="image/webp" sizes="40px" srcset="https://substackcdn.com/image/fetch/$s_!WvgN!,w_40,h_40,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png 40w, https://substackcdn.com/image/fetch/$s_!WvgN!,w_80,h_80,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png 80w, https://substackcdn.com/image/fetch/$s_!WvgN!,w_120,h_120,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png 120w" style="display:none;"/><img style="object-fit:cover;width:40px;height:40px;" src="https://substackcdn.com/image/fetch/$s_!WvgN!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png" srcset="https://substackcdn.com/image/fetch/$s_!WvgN!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png 40w, https://substackcdn.com/image/fetch/$s_!WvgN!,w_80,h_80,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png 80w, https://substackcdn.com/image/fetch/$s_!WvgN!,w_120,h_120,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png 120w" sizes="40px" alt="stories from the edge" draggable="false" class="pencraft pc-reset"/></picture></div></div></a></div><div style="flex-grow:0;" class="titleContainer-DJYq5v"><h1 class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd"><a href="/" class="pencraft pc-display-contents pc-reset">stories from the edge</a></h1></div><div style="flex-basis:0px;flex-grow:1;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Frollback-does-not-erase-distributed&amp;for_pub=storiesfromtheedge" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="height:72px;"></div></div></div><div><script nonce="wKt-w6sK6dzxnR0dSM363A" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://storiesfromtheedge.substack.com/p/rollback-does-not-erase-distributed","mainEntityOfPage":"https://storiesfromtheedge.substack.com/p/rollback-does-not-erase-distributed","headline":"Rollback Does Not Erase Distributed Memory","description":"The rollback finished successfully.","image":[{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!gHJA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png"}],"datePublished":"2026-09-01T08:47:00+00:00","dateModified":"2026-09-01T08:47:00+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"balu Kambala","url":"https://substack.com/@balakambala","description":"Bala Kambala is a backend software engineer with over 15 years of experience designing scalable, secure, and resilient platforms at cloud scale. He is passionate about mentoring engineers, platform evangelism, and shaping technical standards. ","identifier":"user:10924789","image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!PAwz!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!PAwz!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png"}}],"publisher":{"@type":"Organization","name":"stories from the edge","url":"https://storiesfromtheedge.substack.com","description":"My personal Substack","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":1},"identifier":"pub:7084903","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!WvgN!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!WvgN!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!WvgN!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!WvgN!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!WvgN!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!WvgN!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png"}},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":0},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":0},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":0}]}</script><script nonce="wKt-w6sK6dzxnR0dSM363A" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"stories from the edge","item":"https://storiesfromtheedge.substack.com"},{"@type":"ListItem","position":2,"name":"Rollback Does Not Erase Distributed Memory","item":"https://storiesfromtheedge.substack.com/p/rollback-does-not-erase-distributed"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><h1 dir="auto" class="post-title published title-X77sOw">Rollback Does Not Erase Distributed Memory</h1><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-row pc-gap-8 pc-alignItems-center pc-justifyContent-flex-start pc-reset"><div style="--scale:36px;--offset:9px;--border-width:4.5px;" class="pencraft pc-display-flex pc-flexDirection-row pc-alignItems-center pc-justifyContent-flex-start pc-reset ltr-qDBmby"><a href="https://substack.com/@balakambala" aria-label="View balu Kambala's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:36px;" tabindex="0" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6 avatar-u8q6xB last-JfNEJ_"><div style="--scale:36px;" title="balu Kambala" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!PAwz!,w_36,h_36,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png 1x, https://substackcdn.com/image/fetch/$s_!PAwz!,w_72,h_72,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png 2x, https://substackcdn.com/image/fetch/$s_!PAwz!,w_108,h_108,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!PAwz!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png" alt="balu Kambala's avatar" srcset="https://substackcdn.com/image/fetch/$s_!PAwz!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png 1x, https://substackcdn.com/image/fetch/$s_!PAwz!,w_72,h_72,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png 2x, https://substackcdn.com/image/fetch/$s_!PAwz!,w_108,h_108,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png 3x" width="36" height="36" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-reset color-primary-zABazT line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><span style="min-width:0;" data-state="closed"><a href="https://substack.com/@balakambala" class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ">balu Kambala</a></span></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2026-09-01T08:47:00.270Z" title="2026-09-01T08:47:00.270Z">Sep 01, 2026</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://storiesfromtheedge.substack.com/p/rollback-does-not-erase-distributed/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-27" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p><span>The rollback finished successfully. The deployment dashboard showed 100% of nodes running the previous build. The service confirmed the old configuration was active.</span><br/><br/><span>Yet customers were still seeing failures.</span><br/><br/><span>In a distributed system, rolling back code can create a false sense of safety. Reverting the change does not revert every component that has already observed the change. Recursive DNS resolvers and other caches may retain the new state until their TTLs expire. A database may contain records written in a format the previous application cannot read. Queues may still hold work created under the failed version.</span><br/><br/><span>Your servers are running version A, but they are running inside an environment mutated by version B.</span><br/><br/><em>A rollback changes what the source of truth says now. However, it does not erase what the rest of the system has already learned.</em><br/><br/><span>This is why distributed recovery is rarely a single event. Some customers recover immediately. Others recover when caches expire, connections restart, queues drain, or incompatible data is repaired.</span><br/><br/><span>So after a rollback, the important question is not only &quot;Is the old version running?&quot; but also &quot;Which parts of the system still retain state from the new version?&quot; </span></p><p><strong>The Takeaway:</strong><span> Deploying Version 40 over Version 41 changes the code, but not the environment Version 41 mutated. Recovery begins only when the distributed memory such as DNS caches, client pools, and databases is cleared or reconciled.</span></p><p>Slack’s DNSSEC rollout demonstrates what happens when that memory lives in infrastructure you do not own.</p><h2 class="header-anchor-post">The Slack DNSSEC rollback<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§the-slack-dnssec-rollback" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://storiesfromtheedge.substack.com/i/216113519/the-slack-dnssec-rollback" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><span>During its </span><a href="https://slack.engineering/what-happened-during-slacks-dnssec-rollout">DNSSEC rollout </a><span>for </span><a href="http://slack.com">slack.com</a><span> Slack observed some DNS resolvers returning unexpected NODATA responses. The team decided to roll back.</span></p><p>DNSSEC depends on records stored in two places. The ‘.com’ parent zone publishes a DS record telling resolvers that `slack.com` must provide cryptographic proof. Slack’s authoritative zone publishes the matching DNSKEY and RRSIG records.</p><p>Slack removed the DS record from ‘.com’ through its registrar and later disabled signing on its authoritative servers.</p><p>The authoritative configuration looked old again. The resolver caches did not.</p><p>Some recursive resolvers had cached the DS record for up to 24 hours. They still expected signed answers from Slack. When the signing material disappeared, validation failed and those resolvers returned SERVFAIL.</p><p><strong>The rollback made the system worse. </strong><span>Slack had changed the authoritative state, but it could not force resolver caches to forget the DS record.</span></p><p>Slack asked major public DNS providers to flush their caches. Large providers recovered quickly. Other resolvers continued failing until their cached DS records expired.</p><p>As shown below, the rollback completed quickly. Recovery followed a long tail as resolver caches expired on different schedules.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!gHJA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!gHJA!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 424w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 848w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 1272w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!gHJA!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png" width="1456" height="624" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:624,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:206385,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:&quot;https://storiesfromtheedge.substack.com/i/216113519?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png&quot;,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!gHJA!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 424w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 848w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 1272w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image buttonBase-GK1x3M"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon-noB79L"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image buttonBase-GK1x3M"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2 icon-noB79L"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p>Slack’s retained state would eventually disappear as resolver caches expired. Database state is harder: records written by the new version may remain indefinitely.</p><h2 class="header-anchor-post"><strong>The database does not roll back with the code</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§the-database-does-not-roll-back-with-the-code" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://storiesfromtheedge.substack.com/i/216113519/the-database-does-not-roll-back-with-the-code" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><span>CircleCI encountered the </span><a href="https://discuss.circleci.com/t/incident-report-november-8-2021-jobs-stuck-in-a-not-running-state/41890">incident</a><span> in November 2021. A deployment changed the type of a field in its job-distribution database. The new version could not process older jobs, so CircleCI rolled it back. The old version then could not process jobs written during the deployment.</span></p><p>The database now contained two representations, but neither application version understood both.</p><p>Engineers recovered by deploying a compatibility change that ignored the field, then draining the accumulated queues.</p><p>The sequence below shows why neither application version was a safe recovery point. Version B rejected older records, version A rejected records written by version B, and recovery required code that could handle both formats.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!8N05!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!8N05!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 424w, https://substackcdn.com/image/fetch/$s_!8N05!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 848w, https://substackcdn.com/image/fetch/$s_!8N05!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 1272w, https://substackcdn.com/image/fetch/$s_!8N05!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!8N05!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png" width="1456" height="566" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:566,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:405250,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:&quot;https://storiesfromtheedge.substack.com/i/216113519?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png&quot;,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!8N05!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 424w, https://substackcdn.com/image/fetch/$s_!8N05!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 848w, https://substackcdn.com/image/fetch/$s_!8N05!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 1272w, https://substackcdn.com/image/fetch/$s_!8N05!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image buttonBase-GK1x3M"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon-noB79L"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image buttonBase-GK1x3M"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2 icon-noB79L"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p>CircleCI had rolled back the code, but recovery required a compatibility build. Once a new version has written a new representation, the old binary may no longer be a safe recovery target.</p><h2 class="header-anchor-post"><strong>Rollback is not recovery</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§rollback-is-not-recovery" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://storiesfromtheedge.substack.com/i/216113519/rollback-is-not-recovery" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>A dashboard can truthfully report that version 41 was replaced by version 40. That does not mean the system is behaving as version 40 did before the deployment.</p><p>During an incident, I separate three stages:</p><p><span>1. </span><strong>The source was rolled back.</strong><span> The central configuration returned to the old version.</span></p><p><span>2. </span><strong>Components applied the rollback.</strong><span> Systems using that configuration fetched and applied the old version.</span></p><p><span>3. </span><strong>The system recovered.</strong><span> Data and actions created by the new version were made compatible, reversed, or repaired.</span></p><p>Teams often prove the first stage and assume the third. Slack and CircleCI both completed the first stage before reaching the third. Sometimes the safest way to close that gap is to move forward.</p><h2 class="header-anchor-post"><strong>Why roll-forward can be safer</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§why-roll-forward-can-be-safer" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://storiesfromtheedge.substack.com/i/216113519/why-roll-forward-can-be-safer" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>These incidents show that rollback is not always enough for recovery. Once the new version writes incompatible data or publishes state that other systems cache, returning to the old version can create another failure.</p><p>CircleCI recovered by deploying a compatibility build that ignored the changed field. This allowed it to process jobs written by both versions. For Slack, a safer recovery would have kept the signing material available until cached DS records expired.</p><p>The recovery decision should depend on the state already present in the system, not on how easy it is to run the rollback command. The delay between rollback and actual recovery often appears as a long tail in the error graph.</p><h2 class="header-anchor-post"><strong>Reading the recovery curve</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§reading-the-recovery-curve" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://storiesfromtheedge.substack.com/i/216113519/reading-the-recovery-curve" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Slack and CircleCI produced different recovery curves. Slack showed a sharp initial recovery followed by a long tail. CircleCI showed little change after rollback; recovery began only after a compatibility fix and continued as the queue drained.</p><p><span>The curve cannot prove the cause, but it narrows the search. This is what I mean by incident geometry. A sharp recovery may point to a centralized fix. A long tail may indicate components recovering on independent clocks. Little or no improvement after rollback may indicate durable state that requires repair or reconciliation.</span><br/><br/><span>I’ll go deeper into incident geometry—how user-impact shapes can lead us to architectural findings—in my talk, </span><a href="https://www.wearedevelopers.com/events/world-congress-2026-north-america/sessions/1722-the-geometry-of">The Geometry of Incidents: What User-Impact Shapes Reveal About Platform Architecture</a><span>, at WeAreDevelopers World Congress North America. If you’re attending, please stop by. I’d be glad to continue the discussion in person.</span></p><p>
</p></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://storiesfromtheedge.substack.com/p/rollback-does-not-erase-distributed/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-56" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="visibility-check"></div><div id="discussion" class="pencraft pc-display-flex pc-flexDirection-column pc-gap-16 pc-paddingTop-32 pc-paddingBottom-32 pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-32 pc-reset container"><h4 class="pencraft pc-reset line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Discussion about this post</h4><div class="pencraft pc-alignSelf-flex-start pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Select discussion type" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-70-content-comments" data-state="active" id="radix-P0-70-trigger-comments" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Comments</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-70-content-restacks" data-state="inactive" id="radix-P0-70-trigger-restacks" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Restacks</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div></div></div><div id="substack-comments" class="single-post-section comments-section"><div class="container"><div class="visibility-check"></div><div data-test-id="comment-input" class="pencraft pc-display-flex pc-reset flex-grow-rzmknG"><form class="comment-input-form form-CkZ7Kt"><div style="--scale:32px;" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><div style="--scale:32px;" title="User" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-flexDirection-column pc-boxShadow-xs pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-sm"><textarea name="body" placeholder="Write a comment..." aria-label="Write a comment..." rows="4" class="pencraft input-qHk4bN autogrowing-_ipn9Y textarea-GbEjRX inputText-pV_yWb unstyled-IJLUJU"></textarea></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></form></div></div></div></div><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-83-content-top" data-state="active" id="radix-P0-83-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-83-content-new" data-state="inactive" id="radix-P0-83-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." aria-label="Email" type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer themed-background"><div class="container"><div class="footer-terms"><span>© 2026 balu Kambala</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&amp;utm_medium=web&amp;utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&amp;utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="wKt-w6sK6dzxnR0dSM363A">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
</div>
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
<script nonce="wKt-w6sK6dzxnR0dSM363A">
window.Sentry && window.Sentry.onLoad(function() {
window.Sentry.init({
environment: window._preloads.sentry_environment,
dsn: window._preloads.sentry_dsn,
})
})
</script>
<script nonce="wKt-w6sK6dzxnR0dSM363A">window._preloads = JSON.parse("{\"cspNonce\":\"wKt-w6sK6dzxnR0dSM363A\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://storiesfromtheedge.substack.com\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":10924789,\"byline_images_enabled\":true,\"bylines_enabled\":true,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"balu Kambala\",\"cover_photo_url\":null,\"created_at\":\"2025-11-27T04:45:17.444Z\",\"custom_domain_optional\":false,\"custom_domain\":null,\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":null,\"default_show_guest_bios\":true,\"email_banner_url\":null,\"email_from_name\":null,\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":null,\"free_subscription_benefits\":null,\"ga_pixel_id\":null,\"google_site_verification_token\":null,\"google_tag_manager_token\":null,\"hero_image\":null,\"hero_text\":\"My personal Substack\",\"hide_intro_subtitle\":null,\"hide_intro_title\":null,\"hide_podcast_feed_link\":false,\"homepage_type\":\"newspaper\",\"id\":7084903,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":null,\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!WvgN!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png\",\"minimum_group_size\":2,\"moderation_enabled\":true,\"name\":\"stories from the edge\",\"paid_subscription_benefits\":null,\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"disabled\",\"paywall_free_trial_enabled\":false,\"podcast_art_url\":null,\"paid_podcast_episode_art_url\":null,\"podcast_byline\":null,\"podcast_description\":null,\"podcast_enabled\":false,\"podcast_feed_url\":null,\"podcast_title\":null,\"post_preview_limit\":null,\"primary_user_id\":null,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"storiesfromtheedge\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#FF6719\",\"theme_var_color_links\":false,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":null,\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":null,\"stripe_user_id\":null,\"stripe_country\":null,\"stripe_publishable_key\":null,\"stripe_platform_account\":null,\"automatic_tax_enabled\":null,\"fundraising_type\":null,\"author_name\":\"balu Kambala\",\"author_handle\":\"balakambala\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!PAwz!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png\",\"author_bio\":\"Bala Kambala is a backend software engineer with over 15 years of experience designing scalable, secure, and resilient platforms at cloud scale. He is passionate about mentoring engineers, platform evangelism, and shaping technical standards. \",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":null,\"web_bg_color\":null,\"cover_bg_color\":null,\"home_hero\":\"newspaper\",\"home_posts\":\"list\"},\"threads_v2_settings\":null,\"default_group_coupon_percent_off\":null,\"default_group_coupon_include_founding\":null,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":false,\"first_post_date\":\"2025-11-27T05:00:33.704Z\",\"has_podcast\":false,\"has_free_podcast\":false,\"has_subscriber_only_podcast\":false,\"has_community_content\":false,\"rankingDetail\":\"Launched 10 months ago\",\"rankingDetailFreeIncluded\":\"Launched 10 months ago\",\"rankingDetailOrderOfMagnitude\":0,\"rankingDetailFreeIncludedOrderOfMagnitude\":1,\"rankingDetailFreeSubscriberCount\":null,\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u062A\u0645 \u0627\u0644\u0625\u0637\u0644\u0627\u0642 10 months ago\"},\"ca\":{\"rankingDetail\":\"S\u2019ha llan\u00E7at fa 10 mesos\"},\"da\":{\"rankingDetail\":\"Lancering 10 m\u00E5neder\"},\"de\":{\"rankingDetail\":\"vor 10 Monaten gelauncht\"},\"es\":{\"rankingDetail\":\"Lanzado hace 10 meses\"},\"fr\":{\"rankingDetail\":\"Lanc\u00E9 il y a 10 mois\"},\"id\":{\"rankingDetail\":\"Diluncurkan 10 bulan yang lalu\"},\"ja\":{\"rankingDetail\":\"\u958B\u59CB\u65E5 10\u30F6\u6708\u524D\"},\"ko\":{\"rankingDetail\":\"10\uAC1C\uC6D4 \uC804\uC5D0 \uCD9C\uC2DC\"},\"nb\":{\"rankingDetail\":\"Lansert 10 m\u00E5neder\"},\"nl\":{\"rankingDetail\":\"Gelanceerd 10 maanden geleden\"},\"pl\":{\"rankingDetail\":\"Uruchomiono 10 miesi\u0119cy temu\"},\"pt\":{\"rankingDetail\":\"Lan\u00E7ado 10 meses\"},\"pt-br\":{\"rankingDetail\":\"Lan\u00E7ado 10 meses\"},\"en-gb\":{\"rankingDetail\":\"Launched 10 months ago\"},\"it\":{\"rankingDetail\":\"Lanciato 10 mesi\"},\"tr\":{\"rankingDetail\":\"10 ay ba\u015Flat\u0131ld\u0131\"},\"sv\":{\"rankingDetail\":\"Lanserad 10 m\u00E5nader sedan\"},\"fi\":{\"rankingDetail\":\"Launched 10 kuukautta\"},\"is\":{\"rankingDetail\":\"Launched 10 m\u00E1nu\u00F0ir\"},\"zh-hans\":{\"rankingDetail\":\"\u4E8E10 \u4E2A\u6708\u524D\u4E0A\u7EBF\"},\"zh-hant\":{\"rankingDetail\":\"\u65BC 10 \u500B\u6708\u524D \u63A8\u51FA\"},\"en\":{\"rankingDetail\":\"Launched 10 months ago\"}},\"freeSubscriberCount\":null,\"freeSubscriberCountOrderOfMagnitude\":\"8\",\"author_bestseller_tier\":0,\"author_badge\":null,\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":true,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":null,\"no_follow\":true,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"free\",\"sections\":[],\"podcastTabInfo\":null,\"multipub_migration\":null,\"navigationBarItems\":[],\"has_active_perks\":false,\"contributors\":[{\"name\":\"balu Kambala\",\"handle\":\"balakambala\",\"role\":\"admin\",\"owner\":true,\"user_id\":10924789,\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png\",\"bio\":\"Bala Kambala is a backend software engineer with over 15 years of experience designing scalable, secure, and resilient platforms at cloud scale. He is passionate about mentoring engineers, platform evangelism, and shaping technical standards. \",\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"threads_v2_enabled\":false,\"viralGiftsConfig\":null,\"no_index\":false,\"tier\":1,\"can_set_google_site_verification\":false,\"can_have_sitemap\":false,\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://storiesfromtheedge.substack.com\",\"hostname\":\"storiesfromtheedge.substack.com\",\"is_on_substack\":false,\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"pageThemes\":{\"podcast\":null},\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#ff6719\",\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":232,\"g\":83,\"b\":0,\"a\":1},\"primary_elevated\":{\"r\":232,\"g\":83,\"b\":0,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":213,\"g\":66,\"b\":0,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":206.90096477355226,\"g\":206.90096477355175,\"b\":206.9009647735519,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#FF6719\",\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":86,\"b\":0,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.5},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.3},\"accent\":{\"r\":213,\"g\":66,\"b\":0,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1}}},\"bg\":{\"name\":\"None\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}},\"is_dark\":false}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[251,171,36],\"population\":550},\"DarkVibrant\":{\"rgb\":[7,116,92],\"population\":21},\"LightVibrant\":{\"rgb\":[251,108,68],\"population\":640},\"Muted\":{\"rgb\":[112,116,148],\"population\":2},\"DarkMuted\":{\"rgb\":[81,68,55],\"population\":34},\"LightMuted\":{\"rgb\":[159,164,208],\"population\":10}}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":true,\"customDomain\":null},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"just_launched_launch_post_computation\":\"nearest_to_launched_at\",\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"enable_unified_audio_modal\":true,\"enable_survey_response_crm_filters\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"enable_linked_sponsorship_campaign_creatives\":false,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"thebulwark_preview_changes\":\"\",\"web_chat_replies_pane\":false,\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"enable_subscriber_tag_dynamic_content\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"just_launched_launched_at_computation\":\"first_revenue\",\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"enable_high_follower_dm\":false,\"account_based_post_metering_enabled\":false,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_notes_video_cover_editor\":false,\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"apple_pay_merchant_tokens\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"thefp_enable_typography_updates\":false,\"enable_mobile_stats_for_admins\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_livestream_youtube_simulcast\":true,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"thefp_enable_paywall_coupon\":false,\"google_login_enabled\":false,\"apple_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"enable_post_series\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"enable_saved_links\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"gift_article_max_per_period\":5,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"writer_onboarding_recommendations_default_unchecked\":\"experiment\",\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"enable_theme_editor_deferred_saves\":true,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"mobile_user_attachments_enabled\":false,\"launch_assistant_experiment\":\"control\",\"search_retrieval_variant\":\"experiment\",\"publisher_banner\":\"\",\"enable_sponsorship_profile\":false,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"thefp_enable_navigation_redesign\":false,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"tentpole_promo_enabled\":false,\"thefp_homepage_content_blocks_redesign_enabled\":false,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"enable_custom_image_header_option\":false,\"bignight_page_enabled\":false,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_youtube_exploration\":true,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":null,\"twitterPixelId\":null,\"parselyPixelId\":null,\"googleTagManagerToken\":null,\"googleAnalytics4Token\":null,\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":false,\"credit_token_enabled\":false,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":false,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":false,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"gift_articles_enabled\":false,\"google_analytics_4_token\":null,\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"\",\"notes_feed_enabled\":true,\"paywall_unlock_tokens\":false,\"post_preview_crop_gravity\":\"auto\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":false,\"reader_referrals_leaderboard_enabled\":false,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":false,\"local_currency_modal_seen\":false,\"local_payment_methods_modal_seen\":false,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":false,\"welcome_page_opt_out_text\":\"No thanks\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":false,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false,\"fundraising_entity_name\":null,\"fundraising_solicitation_disclaimer\":null,\"fundraising_paid_for_disclaimer\":null},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://storiesfromtheedge.substack.com/p/rollback-does-not-erase-distributed\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":216113519,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2026-09-01T08:47:00.270Z\",\"updated_at\":\"2026-09-18T00:45:41.822Z\",\"publication_id\":7084903,\"search_engine_description\":null,\"search_engine_title\":null,\"section_id\":null,\"should_send_free_preview\":false,\"show_guest_bios\":true,\"slug\":\"rollback-does-not-erase-distributed\",\"social_title\":null,\"subtitle\":\"\",\"teaser_post_eligible\":true,\"title\":\"Rollback Does Not Erase Distributed Memory\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"everyone\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":0,\"reactions\":{\"\u2764\":0},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"cold-restart-resilience\",\"next_post_slug\":null,\"cover_image\":\"https://substackcdn.com/image/fetch/$s_!gHJA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png\",\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":{\"post_id\":216113519,\"podcast_episode_number\":null,\"podcast_season_number\":null,\"podcast_episode_type\":null,\"should_syndicate_to_other_feed\":null,\"syndicate_to_section_id\":null,\"hide_from_feed\":false,\"free_podcast_url\":null,\"free_podcast_duration\":null,\"preview_contains_ad\":false,\"was_imported_self_serve_sync\":false,\"draft_free_podcast_url\":null,\"draft_free_podcast_duration\":null},\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"The rollback finished successfully.\",\"body_html\":\"<p>The rollback finished successfully. The deployment dashboard showed 100% of nodes running the previous build. The service confirmed the old configuration was active.<br><br>Yet customers were still seeing failures.<br><br>In a distributed system, rolling back code can create a false sense of safety. Reverting the change does not revert every component that has already observed the change. Recursive DNS resolvers and other caches may retain the new state until their TTLs expire. A database may contain records written in a format the previous application cannot read. Queues may still hold work created under the failed version.<br><br>Your servers are running version A, but they are running inside an environment mutated by version B.<br><br><em>A rollback changes what the source of truth says now. However, it does not erase what the rest of the system has already learned.</em><br><br>This is why distributed recovery is rarely a single event. Some customers recover immediately. Others recover when caches expire, connections restart, queues drain, or incompatible data is repaired.<br><br>So after a rollback, the important question is not only \\\"Is the old version running?\\\" but also \\\"Which parts of the system still retain state from the new version?\\\" </p><p><strong>The Takeaway:</strong> Deploying Version 40 over Version 41 changes the code, but not the environment Version 41 mutated. Recovery begins only when the distributed memory such as DNS caches, client pools, and databases is cleared or reconciled.</p><p>Slack\u2019s DNSSEC rollout demonstrates what happens when that memory lives in infrastructure you do not own.</p><h2>The Slack DNSSEC rollback</h2><p>During its <a href=\\\"https://slack.engineering/what-happened-during-slacks-dnssec-rollout\\\">DNSSEC rollout </a>for <a href=\\\"http://slack.com\\\">slack.com</a> Slack observed some DNS resolvers returning unexpected NODATA responses. The team decided to roll back.</p><p>DNSSEC depends on records stored in two places. The \u2018.com\u2019 parent zone publishes a DS record telling resolvers that `slack.com` must provide cryptographic proof. Slack\u2019s authoritative zone publishes the matching DNSKEY and RRSIG records.</p><p>Slack removed the DS record from \u2018.com\u2019 through its registrar and later disabled signing on its authoritative servers.</p><p>The authoritative configuration looked old again. The resolver caches did not.</p><p>Some recursive resolvers had cached the DS record for up to 24 hours. They still expected signed answers from Slack. When the signing material disappeared, validation failed and those resolvers returned SERVFAIL.</p><p><strong>The rollback made the system worse. </strong>Slack had changed the authoritative state, but it could not force resolver caches to forget the DS record.</p><p>Slack asked major public DNS providers to flush their caches. Large providers recovered quickly. Other resolvers continued failing until their cached DS records expired.</p><p>As shown below, the rollback completed quickly. Recovery followed a long tail as resolver caches expired on different schedules.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!gHJA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!gHJA!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 424w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 848w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 1272w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!gHJA!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png\\\" width=\\\"1456\\\" height=\\\"624\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:624,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:206385,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:&quot;https://storiesfromtheedge.substack.com/i/216113519?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png&quot;,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!gHJA!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 424w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 848w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 1272w, https://substackcdn.com/image/fetch/$s_!gHJA!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F123af348-cb0e-435b-8896-8403d4e1dc1a_2016x864.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image buttonBase-GK1x3M\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\" class=\\\"icon-noB79L\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image buttonBase-GK1x3M\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2 icon-noB79L\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p>Slack\u2019s retained state would eventually disappear as resolver caches expired. Database state is harder: records written by the new version may remain indefinitely.</p><h2><strong>The database does not roll back with the code</strong></h2><p>CircleCI encountered the <a href=\\\"https://discuss.circleci.com/t/incident-report-november-8-2021-jobs-stuck-in-a-not-running-state/41890\\\">incident</a> in November 2021. A deployment changed the type of a field in its job-distribution database. The new version could not process older jobs, so CircleCI rolled it back. The old version then could not process jobs written during the deployment.</p><p>The database now contained two representations, but neither application version understood both.</p><p>Engineers recovered by deploying a compatibility change that ignored the field, then draining the accumulated queues.</p><p>The sequence below shows why neither application version was a safe recovery point. Version B rejected older records, version A rejected records written by version B, and recovery required code that could handle both formats.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!8N05!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!8N05!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 424w, https://substackcdn.com/image/fetch/$s_!8N05!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 848w, https://substackcdn.com/image/fetch/$s_!8N05!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 1272w, https://substackcdn.com/image/fetch/$s_!8N05!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!8N05!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png\\\" width=\\\"1456\\\" height=\\\"566\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:566,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:405250,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:&quot;https://storiesfromtheedge.substack.com/i/216113519?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png&quot;,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!8N05!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 424w, https://substackcdn.com/image/fetch/$s_!8N05!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 848w, https://substackcdn.com/image/fetch/$s_!8N05!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 1272w, https://substackcdn.com/image/fetch/$s_!8N05!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F21c2703a-5490-4d00-897e-feda88a0fdd7_2304x896.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image buttonBase-GK1x3M\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\" class=\\\"icon-noB79L\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image buttonBase-GK1x3M\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2 icon-noB79L\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p>CircleCI had rolled back the code, but recovery required a compatibility build. Once a new version has written a new representation, the old binary may no longer be a safe recovery target.</p><h2><strong>Rollback is not recovery</strong></h2><p>A dashboard can truthfully report that version 41 was replaced by version 40. That does not mean the system is behaving as version 40 did before the deployment.</p><p>During an incident, I separate three stages:</p><p>1. <strong>The source was rolled back.</strong> The central configuration returned to the old version.</p><p>2. <strong>Components applied the rollback.</strong> Systems using that configuration fetched and applied the old version.</p><p>3. <strong>The system recovered.</strong> Data and actions created by the new version were made compatible, reversed, or repaired.</p><p>Teams often prove the first stage and assume the third. Slack and CircleCI both completed the first stage before reaching the third. Sometimes the safest way to close that gap is to move forward.</p><h2><strong>Why roll-forward can be safer</strong></h2><p>These incidents show that rollback is not always enough for recovery. Once the new version writes incompatible data or publishes state that other systems cache, returning to the old version can create another failure.</p><p>CircleCI recovered by deploying a compatibility build that ignored the changed field. This allowed it to process jobs written by both versions. For Slack, a safer recovery would have kept the signing material available until cached DS records expired.</p><p>The recovery decision should depend on the state already present in the system, not on how easy it is to run the rollback command. The delay between rollback and actual recovery often appears as a long tail in the error graph.</p><h2><strong>Reading the recovery curve</strong></h2><p>Slack and CircleCI produced different recovery curves. Slack showed a sharp initial recovery followed by a long tail. CircleCI showed little change after rollback; recovery began only after a compatibility fix and continued as the queue drained.</p><p>The curve cannot prove the cause, but it narrows the search. This is what I mean by incident geometry. A sharp recovery may point to a centralized fix. A long tail may indicate components recovering on independent clocks. Little or no improvement after rollback may indicate durable state that requires repair or reconciliation.<br><br>I\u2019ll go deeper into incident geometry\u2014how user-impact shapes can lead us to architectural findings\u2014in my talk, <a href=\\\"https://www.wearedevelopers.com/events/world-congress-2026-north-america/sessions/1722-the-geometry-of\\\">The Geometry of Incidents: What User-Impact Shapes Reveal About Platform Architecture</a>, at WeAreDevelopers World Congress North America. If you\u2019re attending, please stop by. I\u2019d be glad to continue the discussion in person.</p><p>\\n</p>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"The rollback finished successfully. The deployment dashboard showed 100% of nodes running the previous build. The service confirmed the old configuration was active.\",\"wordcount\":992,\"post_preview_limit\":null,\"language\":\"en\",\"postTags\":[{\"id\":\"643e396f-38be-4f56-b3a8-3ccfbe3e08f2\",\"publication_id\":7084903,\"name\":\"Outages\",\"slug\":\"outages\",\"hidden\":false},{\"id\":\"ca62b634-d1ca-4637-a06d-fa3253b0a51d\",\"publication_id\":7084903,\"name\":\"SRE\",\"slug\":\"sre\",\"hidden\":false},{\"id\":\"ca71154e-27f5-4914-af1e-d5b018c3f8aa\",\"publication_id\":7084903,\"name\":\"Reliability Engineering\",\"slug\":\"reliability-engineering\",\"hidden\":false}],\"postCountryBlocks\":[],\"headlineTest\":null,\"coverImagePalette\":{\"Vibrant\":{\"rgb\":[108,188,124],\"population\":1},\"DarkVibrant\":{\"rgb\":[41.51495327102804,91.08504672897196,51.428971962616835],\"population\":0},\"LightVibrant\":{\"rgb\":[229,132,132],\"population\":5},\"Muted\":{\"rgb\":[116,124,188],\"population\":2},\"DarkMuted\":{\"rgb\":[47.9018691588785,105.0981308411215,59.341121495327116],\"population\":0},\"LightMuted\":{\"rgb\":[172,212,188],\"population\":1}},\"publishedBylines\":[{\"id\":10924789,\"name\":\"balu Kambala\",\"handle\":\"balakambala\",\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/392eb0e4-ec1c-486b-be22-ace242cb6a68_1024x1024.png\",\"bio\":\"Bala Kambala is a backend software engineer with over 15 years of experience designing scalable, secure, and resilient platforms at cloud scale. He is passionate about mentoring engineers, platform evangelism, and shaping technical standards. \",\"profile_set_up_at\":\"2023-07-17T15:36:45.586Z\",\"reader_installed_at\":\"2023-07-17T15:36:36.152Z\",\"publicationUsers\":[{\"id\":1800061,\"user_id\":10924789,\"publication_id\":1815390,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":1815390,\"name\":\"balu\u2019s Substack\",\"subdomain\":\"balakambala\",\"custom_domain\":null,\"custom_domain_optional\":false,\"hero_text\":\"My personal Substack\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png\",\"author_id\":10924789,\"primary_user_id\":10924789,\"theme_var_background_pop\":\"#A33ACB\",\"created_at\":\"2023-07-19T18:09:47.425Z\",\"email_from_name\":null,\"copyright\":\"balu Kambala\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"newspaper\",\"is_personal_mode\":false,\"logo_url_wide\":null}},{\"id\":7230204,\"user_id\":10924789,\"publication_id\":7084903,\"role\":\"admin\",\"public\":true,\"is_primary\":false,\"publication\":{\"id\":7084903,\"name\":\"stories from the edge\",\"subdomain\":\"storiesfromtheedge\",\"custom_domain\":null,\"custom_domain_optional\":false,\"hero_text\":\"My personal Substack\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/96d68838-6a10-4a0a-80bb-95961010ce21_144x144.png\",\"author_id\":10924789,\"primary_user_id\":null,\"theme_var_background_pop\":\"#FF6719\",\"created_at\":\"2025-11-27T04:45:17.444Z\",\"email_from_name\":null,\"copyright\":\"balu Kambala\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"newspaper\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"is_guest\":false,\"bestseller_tier\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":0,\"comment_count\":0,\"child_comment_count\":0,\"is_geoblocked\":false,\"is_editor_preview\":false,\"hasCashtag\":false},\"comments\":[],\"canonicalUrl\":\"https://storiesfromtheedge.substack.com/p/rollback-does-not-erase-distributed\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://storiesfromtheedge.substack.com/p/rollback-does-not-erase-distributed\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":\"#FF6719\",\"background_pop\":\"#FF6719\",\"color_theme_bg_web\":null,\"cover_bg_color\":\"#FFFFFF\",\"cover_bg_color_secondary\":\"#f0f0f0\",\"background_pop_darken\":\"#ff5600\",\"print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_darken\":\"#ff5600\",\"color_theme_print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_20\":\"rgba(255, 103, 25, 0.2)\",\"color_theme_bg_pop_30\":\"rgba(255, 103, 25, 0.3)\",\"print_pop\":\"#ff6719\",\"color_theme_accent\":\"#ff6719\",\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#ff6719\",\"home_hero\":\"newspaper\",\"home_posts\":\"list\",\"background_contrast_1\":\"#f0f0f0\",\"background_contrast_2\":\"#dddddd\",\"background_contrast_3\":\"#b7b7b7\",\"background_contrast_4\":\"#929292\",\"background_contrast_5\":\"#515151\",\"color_theme_detail\":\"#e6e6e6\",\"background_contrast_pop\":\"rgba(255, 103, 25, 0.4)\",\"color_theme_bg_contrast_pop\":\"rgba(255, 103, 25, 0.4)\",\"theme_bg_is_dark\":\"0\",\"background_pop_rgb\":\"255, 103, 25\",\"color_theme_bg_pop_rgb\":\"255, 103, 25\",\"color_theme_accent_rgb\":\"255, 103, 25\"},\"recentEpisodes\":null,\"postSeriesNav\":null,\"trackFrontendVisit\":true,\"paywallCoupon\":null,\"activeLiveStream\":null,\"freeTrialCoupon\":null,\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"125.0\",\"major\":\"125\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
<script nonce="wKt-w6sK6dzxnR0dSM363A">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"storiesfromtheedge\",\"publication_id\":7084903,\"has_plans\":false,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.2606148d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/8818.207d3931.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/821.a24110e6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3754.28fa1c66.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12.96b5d413.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2732.7a5053a5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/7891.826b782d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/291.ef5d9df6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9742.fd127380.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/6233.990a9dd0.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9614.528e5ac2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/8712.de1dda42.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/6958.d9cb1e1f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4014.9b8c5e64.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9054.57ac7baf.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/1358.15d3b4b9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4807.08163355.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4035.25a67e8d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/535.eb32315f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/8511.4ea00592.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/515.2c928027.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3384.a9bcc9e8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/8914.1998840a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3164.a0690a8b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9115.afc59c56.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/8638.f440202a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/6495.3f95a7fc.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4806.58a34d69.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/195.bd9db349.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/8637.b61bbec0.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/7933.1b28e067.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4232.6c88cc42.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27.90b0c7d0.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/6204.5d7030cb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4141.7f30ccaa.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/1133.27a43ebd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/1737.0b568dbb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/235.95652c59.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4482.8932a9d2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9910.eb0ee58c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3532.8eb1d489.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/6108.23cdb251.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2828.16606cd4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.cb7d8021.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9479.22ff649e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/6168.eb63b758.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5936.435c58fe.js" charset="utf-8"></script>
<script nonce="wKt-w6sK6dzxnR0dSM363A" nomodule>
(function() {
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
var warningDiv = document.createElement('div');
warningDiv.style.color = 'red';
warningDiv.style.padding = '10px';
warningDiv.style.margin = '10px 0';
warningDiv.style.border = '1px solid red';
warningDiv.style.backgroundColor = 'lightyellow';
warningDiv.innerText = message;
document.body.prepend(warningDiv);
})();
</script>
<!-- Datadog Analytics -->
<script nonce="wKt-w6sK6dzxnR0dSM363A">
(function(h,o,u,n,d) {
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
d=o.createElement(u);d.async=1;d.src=n
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
window.DD_RUM.onReady(function() {
window.DD_RUM.init({
clientToken: 'puba71073f072643721169b68f352438710',
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
site: 'datadoghq.com',
service: 'web',
env: window._preloads.dd_env,
version: '5fd83572a06f52b4e06dddaa5d5cb669fa31df58',
sessionSampleRate: 1,
sessionReplaySampleRate: 10,
trackUserInteractions: window._preloads.dd_ti,
trackResources: true,
trackLongTasks: true,
defaultPrivacyLevel: 'mask-user-input',
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
});
})
</script>
<!-- End Datadog Analytics -->
<!-- Cloudflare Web Analytics -->
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
<!-- End Cloudflare Web Analytics -->
<!-- Fallback tracking pixels -->
<noscript>
<style>
#nojs-banner {
position: fixed;
bottom: 0;
left: 0;
padding: 16px 16px 16px 32px;
width: 100%;
box-sizing: border-box;
background: red;
color: white;
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
font-size: 13px;
line-height: 13px;
}
#nojs-banner a {
color: inherit;
text-decoration: underline;
}
</style>
<div id="nojs-banner">
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
</div>
</noscript>
<script nonce="wKt-w6sK6dzxnR0dSM363A">(function(){function c(){var b=a.contentDocument||(a.contentWindow&&a.contentWindow.document);if(b){var d=b.createElement('script');d.nonce='wKt-w6sK6dzxnR0dSM363A';d.innerHTML="window.__CF$cv$params={r:'a426059f8e430d0e',t:'MTc5MDYzMjgzNw=='};var a=document.createElement('script');a.nonce='wKt-w6sK6dzxnR0dSM363A';a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();</script></body>
</html>