1 line
78 KiB
HTML
1 line
78 KiB
HTML
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8" data-next-head=""/><meta name="viewport" content="width=device-width, initial-scale=1" data-next-head=""/><title data-next-head="">How Stripe uses graph search and state machines to auto-remediate a global database fleet | Stripe Dot Dev Blog</title><meta property="og:title" content="How Stripe uses graph search and state machines to auto-remediate a global database fleet" data-next-head=""/><meta name="description" content="Discover how we modeled our MongoDB infrastructure as a traversable graph and then use pathfinding algorithms to dynamically compute and execute recovery plans. This automated approach reduced pager volume by 30% (~200 pages/year), eliminated 12 days of unhealthy shard states annually, and supports new shard layouts with zero manual effort." data-next-head=""/><meta property="og:description" content="Discover how we modeled our MongoDB infrastructure as a traversable graph and then use pathfinding algorithms to dynamically compute and execute recovery plans. This automated approach reduced pager volume by 30% (~200 pages/year), eliminated 12 days of unhealthy shard states annually, and supports new shard layouts with zero manual effort." data-next-head=""/><meta property="og:image" content="https://stripe.dev/dev_social.jpeg" data-next-head=""/><meta property="og:url" content="https://stripe.dev/blog/how-stripe-uses-graph-search-and-state-machines-to-auto-remediate-a-global-database-fleet" data-next-head=""/><meta name="twitter:card" content="summary_large_image" data-next-head=""/><meta name="twitter:site" content="@stripepress" data-next-head=""/><meta name="twitter:creator" content="@stripepress" data-next-head=""/><meta name="twitter:title" content="How Stripe uses graph search and state machines to auto-remediate a global database fleet" data-next-head=""/><meta name="twitter:description" content="Discover how we modeled our MongoDB infrastructure as a traversable graph and then use pathfinding algorithms to dynamically compute and execute recovery plans. This automated approach reduced pager volume by 30% (~200 pages/year), eliminated 12 days of unhealthy shard states annually, and supports new shard layouts with zero manual effort." data-next-head=""/><meta name="twitter:image" content="https://stripe.dev/dev_social.jpeg" data-next-head=""/><link rel="icon" href="../favicon.ico" sizes="16x16" type="image/x-icon" data-next-head=""/><link rel="alternate" type="application/rss+xml" title="Stripe Dot Dev Blog RSS Feed" href="/blog/feed" data-next-head=""/><link rel="preload" href="/_next/static/immutable/chunks/26mgwr_fts6qf.css" as="style"/><link rel="preload" href="/_next/static/immutable/chunks/1noeusy499iw9.css" as="style"/><link rel="preload" href="/_next/static/immutable/chunks/00ox9228fvyp0.css" as="style"/><link rel="stylesheet" href="/_next/static/immutable/chunks/26mgwr_fts6qf.css" data-n-g=""/><link rel="stylesheet" href="/_next/static/immutable/chunks/1noeusy499iw9.css" data-n-p=""/><link rel="stylesheet" href="/_next/static/immutable/chunks/00ox9228fvyp0.css" data-n-p=""/><noscript data-n-css=""></noscript><script src="/_next/static/immutable/chunks/1cob6x-fi4i-4.js" defer=""></script><script src="/_next/static/immutable/chunks/3gsov9ct577rw.js" defer=""></script><script src="/_next/static/immutable/chunks/1ucsp485cb0vt.js" defer=""></script><script src="/_next/static/immutable/chunks/3xifnu-wkzwxk.js" defer=""></script><script src="/_next/static/immutable/chunks/24yokreqzh33m.js" defer=""></script><script src="/_next/static/immutable/chunks/turbopack-40ownz5n31cvn.js" defer=""></script><script src="/_next/static/immutable/chunks/2yxzj6ro62a4z.js" defer=""></script><script src="/_next/static/immutable/chunks/0nqa9reizfal_.js" defer=""></script><script src="/_next/static/immutable/chunks/1ysqz5d0mpqb9.js" defer=""></script><script src="/_next/static/immutable/chunks/1n6eo0j0ma__p.js" defer=""></script><script src="/_next/static/immutable/chunks/1ohutym-kn4k_.js" defer=""></script><script src="/_next/static/immutable/chunks/turbopack-2c2kus_j-wlxb.js" defer=""></script><script src="/_next/static/CpqpGfDouU7Dr4XKHLZDI/_buildManifest.js" defer=""></script><script src="/_next/static/CpqpGfDouU7Dr4XKHLZDI/_ssgManifest.js" defer=""></script><script src="/_next/static/CpqpGfDouU7Dr4XKHLZDI/_clientMiddlewareManifest.js" defer=""></script></head><body><div id="__next"><main><div class="GridLayout-module__h3nX9W__grid"><nav class="Nav-module__bLoQaq__nav"><div class="Nav-module__bLoQaq__nav__left"><a class="Nav-module__bLoQaq__nav__link" aria-label="Home" href="/"><svg class="Nav-module__bLoQaq__nav__logoIcon" width="13" height="15" viewBox="0 0 13 15" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path fill-rule="evenodd" clip-rule="evenodd" d="M0 14.0578L12.6296 11.302V0.942383L0 3.64711V14.0578Z" fill="currentColor"></path></svg></a><a target="_self" class="text-smallcaps NavButton-module__Vo4Iva__navButton NavButton-module__Vo4Iva__active" href="/blog"><span class="NavButton-module__Vo4Iva__hotkey">[<!-- -->B<!-- -->] </span>Blog</a><a target="_self" class="text-smallcaps NavButton-module__Vo4Iva__navButton" href="/events"><span class="NavButton-module__Vo4Iva__hotkey">[<!-- -->E<!-- -->] </span>Events</a><a target="_blank" class="text-smallcaps NavButton-module__Vo4Iva__navButton" href="https://docs.stripe.com"><span class="NavButton-module__Vo4Iva__hotkey">[<!-- -->D<!-- -->] </span>Docs</a><a target="_blank" class="text-smallcaps NavButton-module__Vo4Iva__navButton" href="https://www.youtube.com/stripedevelopers"><span class="NavButton-module__Vo4Iva__hotkey">[<!-- -->Y<!-- -->] </span>Youtube</a><a target="_blank" class="text-smallcaps NavButton-module__Vo4Iva__navButton" href="https://github.com/stripe"><span class="NavButton-module__Vo4Iva__hotkey">[<!-- -->G<!-- -->] </span>Github</a><a target="_blank" class="text-smallcaps NavButton-module__Vo4Iva__navButton" href="https://www.meetup.com/pro/stripe"><span class="NavButton-module__Vo4Iva__hotkey">[<!-- -->M<!-- -->] </span>Meetups</a><a target="_blank" class="text-smallcaps NavButton-module__Vo4Iva__navButton" href="https://www.stripecommunity.com/home"><span class="NavButton-module__Vo4Iva__hotkey">[<!-- -->U<!-- -->] </span>Community</a></div></nav><section class="section-module__Yglhya__section BlogPostHero-module__AQEnuW__section"><div class="backgroundIcons-module__vHa6oa__backgroundIcons"><svg class="BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg></div><h1 class="BlogPostHero-module__AQEnuW__title">How Stripe uses graph search and state machines to auto-remediate a global database fleet</h1><div class="BlogPostHero-module__AQEnuW__icon"><div class="GlobeIcon-module__AdM6KG__icon"><canvas class="GlobeIcon-module__AdM6KG__ellipseCanvas"></canvas><svg width="85" height="42" viewBox="0 0 83 42" fill="none"><circle cx="63" cy="21" r="19.5" stroke="var(--globeGraphicColor)"></circle><circle cx="19.5" cy="21" r="19.5" stroke="var(--globeGraphicColor)"></circle><path d="M3.5 9.75 35 9.75" stroke="var(--globeGraphicColor)"></path><path d="M0 21 39 21" stroke="var(--globeGraphicColor)"></path><path d="M3 32 35 32.5" stroke="var(--globeGraphicColor)"></path><path d="M54.096 16.056H57.948V16.788H58.68V17.532H59.424V24.336H58.68V25.08H57.948V25.836H54.096V16.056ZM57.816 24.948V24.216H58.56V17.664H57.816V16.92H54.984V24.948H57.816ZM60.336 16.056H65.664V16.92H61.224V20.496H64.92V21.372H61.224V24.948H65.664V25.836H60.336V16.056ZM68.076 22.116H67.332V19.152H66.576V16.056H67.464V19.02H68.196V21.984H68.94V24.216H69.552V21.984H70.296V19.02H71.04V16.056H71.904V19.152H71.16V22.116H70.428V24.336H69.684V25.836H68.808V24.336H68.076V22.116Z" fill="var(--globeGraphicColor)"></path></svg></div></div><div class="backgroundIcons-module__vHa6oa__backgroundIcons"><svg class="BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg></div></section><section class="section-module__Yglhya__section BlogPost-module__ifVEqa__blogPostSection"><div class="BlogPost-module__ifVEqa__sidebar"><div class="BlogPost-module__ifVEqa__metadata"><div class="TableHeader-module__qdaoaq__container BlogPost-module__ifVEqa__metadataHeader"><div class="TableHeader-module__qdaoaq__label text-smallcaps"><span>/</span>Metadata</div></div><div class="BlogPost-module__ifVEqa__metadata"><div class="text-smallcaps BlogPost-module__ifVEqa__metaDataItem"><span>Date:</span><span>2026.7.16</span></div><div class="text-smallcaps BlogPost-module__ifVEqa__metaDataItem"><span>Authors:</span><div class="BlogPost-module__ifVEqa__tags"><a href="/authors/pragya-mehta" class="text-smallcaps Tag-module__zQcxqq__tag">Pragya Mehta</a><a href="/authors/sai-samant" class="text-smallcaps Tag-module__zQcxqq__tag">Sai Samant</a></div></div><div class="text-smallcaps BlogPost-module__ifVEqa__metaDataItem"><span>Reading time:</span><span>12 min read</span></div><div class="text-smallcaps BlogPost-module__ifVEqa__metaDataItem"><span>Categories:</span><div class="BlogPost-module__ifVEqa__tags"><div class="text-smallcaps Tag-module__zQcxqq__tag">Engineering</div></div></div><div class="BlogPost-module__ifVEqa__metaDataShare"><span class="text-smallcaps">Share:</span><div class="BlogPost-module__ifVEqa__shareButtonContainer"><a target="_blank" class="buttonSm" href="https://twitter.com/intent/tweet?url=https://stripe.dev/blog/how-stripe-uses-graph-search-and-state-machines-to-auto-remediate-a-global-database-fleet&text=How Stripe uses graph search and state machines to auto-remediate a global database fleet">Twitter/X</a><a target="_blank" class="buttonSm" href="http://www.linkedin.com/shareArticle?mini=true&url=https://stripe.dev/blog/how-stripe-uses-graph-search-and-state-machines-to-auto-remediate-a-global-database-fleet&title=How%20Stripe%20uses%20graph%20search%20and%20state%20machines%20to%20auto-remediate%20a%20global%20database%20fleet&summary=Discover%20how%20we%20modeled%20our%20MongoDB%20infrastructure%20as%20a%20traversable%20graph%20and%20then%20use%20pathfinding%20algorithms%20to%20dynamically%20compute%20and%20execute%20recovery%20plans.%20This%20automated%20approach%20reduced%20pager%20volume%20by%2030%25%20(~200%20pages%2Fyear)%2C%20eliminated%2012%20days%20of%20unhealthy%20shard%20states%20annually%2C%20and%20supports%20new%20shard%20layouts%20with%20zero%20manual%20effort.">LinkedIn</a></div></div></div></div></div><div class="BlogPost-module__ifVEqa__article"><div class="TableHeader-module__qdaoaq__container BlogPost-module__ifVEqa__articleHeader"><div class="TableHeader-module__qdaoaq__label text-smallcaps"><span>/</span>Article</div></div><div class="BlogPost-module__ifVEqa__articleBody"></div><div class="BlogPost-module__ifVEqa__aboutAuthor"><h2 class="BlogPost-module__ifVEqa__srOnly">About the author<!-- -->s</h2><div class="TableHeader-module__qdaoaq__container BlogPost-module__ifVEqa__aboutAuthorHeader"><div class="TableHeader-module__qdaoaq__label text-smallcaps"><span>/</span>About the author<!-- -->s</div></div><div class="BlogPost-module__ifVEqa__authorContent"><h3 class="text-lg">Pragya Mehta</h3><p class="text-sm">Pragya Mehta is an engineer on the Document Database Control Plane Platform team at Stripe</p><ul class="BlogPost-module__ifVEqa__additionalResourcesList"></ul></div><div class="BlogPost-module__ifVEqa__authorContent"><h3 class="text-lg">Sai Samant</h3><p class="text-sm">Sai Samant is a technical writer at Stripe working across the engineering organization.</p><ul class="BlogPost-module__ifVEqa__additionalResourcesList"><li class="ListItem-module__kBFJiq__listItem"><a class="BlogPost-module__ifVEqa__link" href="/blog/extending-stripes-network-with-stablecoins"><span class="text-smallcaps BlogPost-module__ifVEqa__label"><div class="SquareIcon-module__6K7DfG__icon"></div>Extending Stripe’s network with stablecoins</span><svg class="BlogPost-module__ifVEqa__arrow ArrowIcon-module__56PCEG__icon" viewBox="0 0 7 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 0H3V1H4V2H5V3H0V4H5V5H4V6H3V7H4V6H5V5H6V4H7V3H6V2H5V1H4V0Z" fill="currentColor"></path></svg></a></li><li class="ListItem-module__kBFJiq__listItem"><a class="BlogPost-module__ifVEqa__link" href="/blog/how-we-built-stripe-credits-a-programmable-auditable-way-to-pay-your-stripe-fees"><span class="text-smallcaps BlogPost-module__ifVEqa__label"><div class="SquareIcon-module__6K7DfG__icon"></div>How we built Stripe Credits: A programmable, auditable way to pay your Stripe fees</span><svg class="BlogPost-module__ifVEqa__arrow ArrowIcon-module__56PCEG__icon" viewBox="0 0 7 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 0H3V1H4V2H5V3H0V4H5V5H4V6H3V7H4V6H5V5H6V4H7V3H6V2H5V1H4V0Z" fill="currentColor"></path></svg></a></li></ul></div></div><div class="BlogPost-module__ifVEqa__additionalResources"><div class="TableHeader-module__qdaoaq__container BlogPost-module__ifVEqa__additionalResourcesHeader"><div class="TableHeader-module__qdaoaq__label text-smallcaps"><span>/</span>Additional resources</div></div><ul class="BlogPost-module__ifVEqa__additionalResourcesList"><li class="ListItem-module__kBFJiq__listItem"><a class="BlogPost-module__ifVEqa__link" target="_blank" href="https://www.youtube.com/stripedevelopers"><span class="text-smallcaps BlogPost-module__ifVEqa__label"><div class="SquareIcon-module__6K7DfG__icon"></div>Subscribe to Stripe Developers on YouTube.</span><svg class="BlogPost-module__ifVEqa__arrow ArrowIcon-module__56PCEG__icon" viewBox="0 0 7 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 0H3V1H4V2H5V3H0V4H5V5H4V6H3V7H4V6H5V5H6V4H7V3H6V2H5V1H4V0Z" fill="currentColor"></path></svg></a></li><li class="ListItem-module__kBFJiq__listItem"><a class="BlogPost-module__ifVEqa__link" target="_blank" href="https://docs.stripe.com/"><span class="text-smallcaps BlogPost-module__ifVEqa__label"><div class="SquareIcon-module__6K7DfG__icon"></div>Check out the docs for the in-depth developer guidance.</span><svg class="BlogPost-module__ifVEqa__arrow ArrowIcon-module__56PCEG__icon" viewBox="0 0 7 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 0H3V1H4V2H5V3H0V4H5V5H4V6H3V7H4V6H5V5H6V4H7V3H6V2H5V1H4V0Z" fill="currentColor"></path></svg></a></li><li class="ListItem-module__kBFJiq__listItem"><a class="BlogPost-module__ifVEqa__link" target="_blank" href="https://discord.com/invite/RuJnSBXrQn"><span class="text-smallcaps BlogPost-module__ifVEqa__label"><div class="SquareIcon-module__6K7DfG__icon"></div>Join the Stripe Discord server to chat live with other developers.</span><svg class="BlogPost-module__ifVEqa__arrow ArrowIcon-module__56PCEG__icon" viewBox="0 0 7 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 0H3V1H4V2H5V3H0V4H5V5H4V6H3V7H4V6H5V5H6V4H7V3H6V2H5V1H4V0Z" fill="currentColor"></path></svg></a></li><li class="ListItem-module__kBFJiq__listItem"><a class="BlogPost-module__ifVEqa__link" target="_blank" href="https://www.meetup.com/pro/stripe/"><span class="text-smallcaps BlogPost-module__ifVEqa__label"><div class="SquareIcon-module__6K7DfG__icon"></div>Join a local Stripe Developer Meetup to learn about the latest features and network with your community.</span><svg class="BlogPost-module__ifVEqa__arrow ArrowIcon-module__56PCEG__icon" viewBox="0 0 7 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 0H3V1H4V2H5V3H0V4H5V5H4V6H3V7H4V6H5V5H6V4H7V3H6V2H5V1H4V0Z" fill="currentColor"></path></svg></a></li></ul></div></div></section><section class="section-module__Yglhya__section BlogPost-module__ifVEqa__relatedArticlesSection"><div class="TableHeader-module__qdaoaq__container BlogPost-module__ifVEqa__relatedHeader"><div class="TableHeader-module__qdaoaq__label text-smallcaps"><span>/</span>Related Articles</div></div><div class="FeaturedPost-module__8Dh16G__post" data-testid="featured-post-card"><div class="Art-module__QRr6PW__Art FeaturedPost-module__8Dh16G__image" data-testid="artwork" draggable="false" style="-webkit-touch-callout:none;-webkit-user-select:none;user-select:none;touch-action:pan-y"><div class="Art-module__QRr6PW__Chrome"><div class="Art-module__QRr6PW__dragline"></div><div class="Art-module__QRr6PW__fig text-smallcaps">[ <!-- -->Fig. 1<!-- --> ]</div><div class="Art-module__QRr6PW__dragline"></div><a href="/art/index.html?config=%7B%22ampX%22:0.34922884912043817,%22ampY%22:0.366142391782254,%22aspectRatio%22:3.5991310963975267,%22axis%22:%22xy%22,%22lump%22:0.33172155507281426,%22count%22:70,%22freq%22:0.4,%22isDial%22:false,%22isLineart%22:false,%22isRing%22:true,%22isSpiral%22:true,%22left%22:4,%22mouseX%22:0.5866846039371565,%22mouseY%22:0.13641974309831856,%22noise%22:0.08586734730005265,%22scale%22:6.598315318033099,%22shape%22:%22rect%22,%22top%22:12,%22twirl%22:0,%22twist%22:1.4917267701253296,%22velocity%22:2.061707329314202%7D" target="_blank" aria-label="Open art controls"><svg class="Art-module__QRr6PW__propsIcon" width="12" height="12" viewBox="0 0 12 12" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path fill-rule="evenodd" clip-rule="evenodd" d="M-8.74228e-08 10L8 10L8 9L11 9L11 10L12 10L12 11L11 11L11 12L8 12L8 11L-4.37113e-08 11L-8.74228e-08 10ZM-3.93402e-07 3L-4.37114e-07 2L8 2L8 1L11 1L11 2L12 2L12 3L11 3L11 4L8 4L8 3L-3.93402e-07 3ZM1 6L-2.62268e-07 6L-2.18557e-07 7L1 7L1 8L4 8L4 7L12 7L12 6L4 6L4 5L1 5L1 6ZM9 2L10 2L10 3L9 3L9 2ZM10 10L9 10L9 11L10 11L10 10ZM2 6L3 6L3 7L2 7L2 6Z"></path></svg></a></div><div class="Art-module__QRr6PW__row2"><div class="Art-module__QRr6PW__left"></div><div class="Art-module__QRr6PW__canvasContainer"><div class="Art-module__QRr6PW__background"><svg width="326" height="276" viewBox="0 0 326 276" xmlns="http://www.w3.org/2000/svg"><circle cx="13.5" cy="12" r="1"></circle><circle cx="13.5" cy="34.8182" r="1"></circle><circle cx="13.5" cy="57.6364" r="1"></circle><circle cx="13.5" cy="80.4545" r="1"></circle><circle cx="13.5" cy="103.273" r="1"></circle><circle cx="13.5" cy="126.091" r="1"></circle><circle cx="13.5" cy="148.909" r="1"></circle><circle cx="13.5" cy="171.727" r="1"></circle><circle cx="13.5" cy="194.545" r="1"></circle><circle cx="13.5" cy="217.364" r="1"></circle><circle cx="13.5" cy="240.182" r="1"></circle><circle cx="13.5" cy="263" r="1"></circle><circle cx="33.5" cy="12" r="1"></circle><circle cx="33.5" cy="34.8182" r="1"></circle><circle cx="33.5" cy="57.6364" r="1"></circle><circle cx="33.5" cy="80.4545" r="1"></circle><circle cx="33.5" cy="103.273" r="1"></circle><circle cx="33.5" cy="126.091" r="1"></circle><circle cx="33.5" cy="148.909" r="1"></circle><circle cx="33.5" cy="171.727" r="1"></circle><circle cx="33.5" cy="194.545" r="1"></circle><circle cx="33.5" cy="217.364" r="1"></circle><circle cx="33.5" cy="240.182" r="1"></circle><circle cx="33.5" cy="263" r="1"></circle><circle cx="53.5" cy="12" r="1"></circle><circle cx="53.5" cy="34.8182" r="1"></circle><circle cx="53.5" cy="57.6364" r="1"></circle><circle cx="53.5" cy="80.4545" r="1"></circle><circle cx="53.5" cy="103.273" r="1"></circle><circle cx="53.5" cy="126.091" r="1"></circle><circle cx="53.5" cy="148.909" r="1"></circle><circle cx="53.5" cy="171.727" r="1"></circle><circle cx="53.5" cy="194.545" r="1"></circle><circle cx="53.5" cy="217.364" r="1"></circle><circle cx="53.5" cy="240.182" r="1"></circle><circle cx="53.5" cy="263" r="1"></circle><circle cx="73.5" cy="12" r="1"></circle><circle cx="73.5" cy="34.8182" r="1"></circle><circle cx="73.5" cy="57.6364" r="1"></circle><circle cx="73.5" cy="80.4545" r="1"></circle><circle cx="73.5" cy="103.273" r="1"></circle><circle cx="73.5" cy="126.091" r="1"></circle><circle cx="73.5" cy="148.909" r="1"></circle><circle cx="73.5" cy="171.727" r="1"></circle><circle cx="73.5" cy="194.545" r="1"></circle><circle cx="73.5" cy="217.364" r="1"></circle><circle cx="73.5" cy="240.182" r="1"></circle><circle cx="73.5" cy="263" r="1"></circle><circle cx="93.5" cy="12" r="1"></circle><circle cx="93.5" cy="34.8182" r="1"></circle><circle cx="93.5" cy="57.6364" r="1"></circle><circle cx="93.5" cy="80.4545" r="1"></circle><circle cx="93.5" cy="103.273" r="1"></circle><circle cx="93.5" cy="126.091" r="1"></circle><circle cx="93.5" cy="148.909" r="1"></circle><circle cx="93.5" cy="171.727" r="1"></circle><circle cx="93.5" cy="194.545" r="1"></circle><circle cx="93.5" cy="217.364" r="1"></circle><circle cx="93.5" cy="240.182" r="1"></circle><circle cx="93.5" cy="263" r="1"></circle><circle cx="113.5" cy="12" r="1"></circle><circle cx="113.5" cy="34.8182" r="1"></circle><circle cx="113.5" cy="57.6364" r="1"></circle><circle cx="113.5" cy="80.4545" r="1"></circle><circle cx="113.5" cy="103.273" r="1"></circle><circle cx="113.5" cy="126.091" r="1"></circle><circle cx="113.5" cy="148.909" r="1"></circle><circle cx="113.5" cy="171.727" r="1"></circle><circle cx="113.5" cy="194.545" r="1"></circle><circle cx="113.5" cy="217.364" r="1"></circle><circle cx="113.5" cy="240.182" r="1"></circle><circle cx="113.5" cy="263" r="1"></circle><circle cx="133.5" cy="12" r="1"></circle><circle cx="133.5" cy="34.8182" r="1"></circle><circle cx="133.5" cy="57.6364" r="1"></circle><circle cx="133.5" cy="80.4545" r="1"></circle><circle cx="133.5" cy="103.273" r="1"></circle><circle cx="133.5" cy="126.091" r="1"></circle><circle cx="133.5" cy="148.909" r="1"></circle><circle cx="133.5" cy="171.727" r="1"></circle><circle cx="133.5" cy="194.545" r="1"></circle><circle cx="133.5" cy="217.364" r="1"></circle><circle cx="133.5" cy="240.182" r="1"></circle><circle cx="133.5" cy="263" r="1"></circle><circle cx="153.5" cy="12" r="1"></circle><circle cx="153.5" cy="34.8182" r="1"></circle><circle cx="153.5" cy="57.6364" r="1"></circle><circle cx="153.5" cy="80.4545" r="1"></circle><circle cx="153.5" cy="103.273" r="1"></circle><circle cx="153.5" cy="126.091" r="1"></circle><circle cx="153.5" cy="148.909" r="1"></circle><circle cx="153.5" cy="171.727" r="1"></circle><circle cx="153.5" cy="194.545" r="1"></circle><circle cx="153.5" cy="217.364" r="1"></circle><circle cx="153.5" cy="240.182" r="1"></circle><circle cx="153.5" cy="263" r="1"></circle><circle cx="173.5" cy="12" r="1"></circle><circle cx="173.5" cy="34.8182" r="1"></circle><circle cx="173.5" cy="57.6364" r="1"></circle><circle cx="173.5" cy="80.4545" r="1"></circle><circle cx="173.5" cy="103.273" r="1"></circle><circle cx="173.5" cy="126.091" r="1"></circle><circle cx="173.5" cy="148.909" r="1"></circle><circle cx="173.5" cy="171.727" r="1"></circle><circle cx="173.5" cy="194.545" r="1"></circle><circle cx="173.5" cy="217.364" r="1"></circle><circle cx="173.5" cy="240.182" r="1"></circle><circle cx="173.5" cy="263" r="1"></circle><circle cx="193.5" cy="12" r="1"></circle><circle cx="193.5" cy="34.8182" r="1"></circle><circle cx="193.5" cy="57.6364" r="1"></circle><circle cx="193.5" cy="80.4545" r="1"></circle><circle cx="193.5" cy="103.273" r="1"></circle><circle cx="193.5" cy="126.091" r="1"></circle><circle cx="193.5" cy="148.909" r="1"></circle><circle cx="193.5" cy="171.727" r="1"></circle><circle cx="193.5" cy="194.545" r="1"></circle><circle cx="193.5" cy="217.364" r="1"></circle><circle cx="193.5" cy="240.182" r="1"></circle><circle cx="193.5" cy="263" r="1"></circle><circle cx="213.5" cy="12" r="1"></circle><circle cx="213.5" cy="34.8182" r="1"></circle><circle cx="213.5" cy="57.6364" r="1"></circle><circle cx="213.5" cy="80.4545" r="1"></circle><circle cx="213.5" cy="103.273" r="1"></circle><circle cx="213.5" cy="126.091" r="1"></circle><circle cx="213.5" cy="148.909" r="1"></circle><circle cx="213.5" cy="171.727" r="1"></circle><circle cx="213.5" cy="194.545" r="1"></circle><circle cx="213.5" cy="217.364" r="1"></circle><circle cx="213.5" cy="240.182" r="1"></circle><circle cx="213.5" cy="263" r="1"></circle><circle cx="233.5" cy="12" r="1"></circle><circle cx="233.5" cy="34.8182" r="1"></circle><circle cx="233.5" cy="57.6364" r="1"></circle><circle cx="233.5" cy="80.4545" r="1"></circle><circle cx="233.5" cy="103.273" r="1"></circle><circle cx="233.5" cy="126.091" r="1"></circle><circle cx="233.5" cy="148.909" r="1"></circle><circle cx="233.5" cy="171.727" r="1"></circle><circle cx="233.5" cy="194.545" r="1"></circle><circle cx="233.5" cy="217.364" r="1"></circle><circle cx="233.5" cy="240.182" r="1"></circle><circle cx="233.5" cy="263" r="1"></circle><circle cx="253.5" cy="12" r="1"></circle><circle cx="253.5" cy="34.8182" r="1"></circle><circle cx="253.5" cy="57.6364" r="1"></circle><circle cx="253.5" cy="80.4545" r="1"></circle><circle cx="253.5" cy="103.273" r="1"></circle><circle cx="253.5" cy="126.091" r="1"></circle><circle cx="253.5" cy="148.909" r="1"></circle><circle cx="253.5" cy="171.727" r="1"></circle><circle cx="253.5" cy="194.545" r="1"></circle><circle cx="253.5" cy="217.364" r="1"></circle><circle cx="253.5" cy="240.182" r="1"></circle><circle cx="253.5" cy="263" r="1"></circle><circle cx="273.5" cy="12" r="1"></circle><circle cx="273.5" cy="34.8182" r="1"></circle><circle cx="273.5" cy="57.6364" r="1"></circle><circle cx="273.5" cy="80.4545" r="1"></circle><circle cx="273.5" cy="103.273" r="1"></circle><circle cx="273.5" cy="126.091" r="1"></circle><circle cx="273.5" cy="148.909" r="1"></circle><circle cx="273.5" cy="171.727" r="1"></circle><circle cx="273.5" cy="194.545" r="1"></circle><circle cx="273.5" cy="217.364" r="1"></circle><circle cx="273.5" cy="240.182" r="1"></circle><circle cx="273.5" cy="263" r="1"></circle><circle cx="293.5" cy="12" r="1"></circle><circle cx="293.5" cy="34.8182" r="1"></circle><circle cx="293.5" cy="57.6364" r="1"></circle><circle cx="293.5" cy="80.4545" r="1"></circle><circle cx="293.5" cy="103.273" r="1"></circle><circle cx="293.5" cy="126.091" r="1"></circle><circle cx="293.5" cy="148.909" r="1"></circle><circle cx="293.5" cy="171.727" r="1"></circle><circle cx="293.5" cy="194.545" r="1"></circle><circle cx="293.5" cy="217.364" r="1"></circle><circle cx="293.5" cy="240.182" r="1"></circle><circle cx="293.5" cy="263" r="1"></circle><circle cx="313.5" cy="12" r="1"></circle><circle cx="313.5" cy="34.8182" r="1"></circle><circle cx="313.5" cy="57.6364" r="1"></circle><circle cx="313.5" cy="80.4545" r="1"></circle><circle cx="313.5" cy="103.273" r="1"></circle><circle cx="313.5" cy="126.091" r="1"></circle><circle cx="313.5" cy="148.909" r="1"></circle><circle cx="313.5" cy="171.727" r="1"></circle><circle cx="313.5" cy="194.545" r="1"></circle><circle cx="313.5" cy="217.364" r="1"></circle><circle cx="313.5" cy="240.182" r="1"></circle><circle cx="313.5" cy="263" r="1"></circle></svg></div><canvas class="Art-module__QRr6PW__Canvas"></canvas><div class="Art-module__QRr6PW__tenX"><svg width="9" height="7" viewBox="0 0 9 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M0 7V0H1V1H2V2H3V3H4V4H3V5H2V6H1V7H0Z"></path><path d="M5 7V0H6V1H7V2H8V3H9V4H8V5H7V6H6V7H5Z"></path></svg><div class="text-smallcaps">10x</div></div></div><div class="Art-module__QRr6PW__right"></div></div><div class="Art-module__QRr6PW__bottom"></div></div><div class="FeaturedPost-module__8Dh16G__copy"><a class="FeaturedPost-module__8Dh16G__copyLink" target="_blank" href="/blog/building-a-data-plane-from-scratch-stripes-own-high-performance-distributed-proxy"><div class="text-md FeaturedPost-module__8Dh16G__title" data-testid="featured-post-title"><span class="FeaturedPost-module__8Dh16G__highlight">Building a data plane from scratch: Stripe’s own high-performance distributed proxy<span class="FeaturedPost-module__8Dh16G__arrow"><svg viewBox="0 0 6 6" fill="none" xmlns="http://www.w3.org/2000/svg"><rect y="5" width="1" height="1" fill="currentColor"></rect><rect x="1" y="4" width="1" height="1" fill="currentColor"></rect><rect x="2" y="3" width="1" height="1" fill="currentColor"></rect><rect x="3" y="2" width="1" height="1" fill="currentColor"></rect><rect x="4" y="1" width="1" height="1" fill="currentColor"></rect><rect x="5" width="1" height="6" fill="currentColor"></rect><rect width="6" height="1" fill="currentColor"></rect></svg></span></span></div><div class="text-sm FeaturedPost-module__8Dh16G__body" data-testid="featured-post-summary"><span class="FeaturedPost-module__8Dh16G__bodyHighlight">To minimize failed payments for our users, Stripe targets 99.9995% reliability. As traffic grew beyond 1.6% of global GDP, this put intense strain...</span><div class="FeaturedPost-module__8Dh16G__tags"><div class="text-smallcaps Tag-module__zQcxqq__tag FeaturedPost-module__8Dh16G__tagHighlight">Engineering</div><div class="text-smallcaps Tag-module__zQcxqq__tag FeaturedPost-module__8Dh16G__tagHighlight">Infrastructure</div></div></div></a></div></div><div class="FeaturedPost-module__8Dh16G__post FeaturedPost-module__8Dh16G__separator" data-testid="featured-post-card"><div class="Art-module__QRr6PW__Art FeaturedPost-module__8Dh16G__image" data-testid="artwork" draggable="false" style="-webkit-touch-callout:none;-webkit-user-select:none;user-select:none;touch-action:pan-y"><div class="Art-module__QRr6PW__Chrome"><div class="Art-module__QRr6PW__dragline"></div><div class="Art-module__QRr6PW__fig text-smallcaps">[ <!-- -->Fig. 2<!-- --> ]</div><div class="Art-module__QRr6PW__dragline"></div><a href="/art/index.html?config=%7B%22ampX%22:0.45312970344722275,%22ampY%22:0.6594387119449675,%22aspectRatio%22:3.640611987476796,%22axis%22:%22xy%22,%22lump%22:0.6013426144607366,%22count%22:104,%22freq%22:0.501017894551158,%22isDial%22:false,%22isLineart%22:false,%22isRing%22:true,%22isSpiral%22:true,%22left%22:-23,%22mouseX%22:0.5907904613740742,%22mouseY%22:0.1359473072765395,%22noise%22:0.3699935179296881,%22scale%22:7.06511910136044,%22shape%22:%22rect%22,%22top%22:-18,%22twirl%22:0,%22twist%22:7.786044806666672,%22velocity%22:2.002865631341934%7D" target="_blank" aria-label="Open art controls"><svg class="Art-module__QRr6PW__propsIcon" width="12" height="12" viewBox="0 0 12 12" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path fill-rule="evenodd" clip-rule="evenodd" d="M-8.74228e-08 10L8 10L8 9L11 9L11 10L12 10L12 11L11 11L11 12L8 12L8 11L-4.37113e-08 11L-8.74228e-08 10ZM-3.93402e-07 3L-4.37114e-07 2L8 2L8 1L11 1L11 2L12 2L12 3L11 3L11 4L8 4L8 3L-3.93402e-07 3ZM1 6L-2.62268e-07 6L-2.18557e-07 7L1 7L1 8L4 8L4 7L12 7L12 6L4 6L4 5L1 5L1 6ZM9 2L10 2L10 3L9 3L9 2ZM10 10L9 10L9 11L10 11L10 10ZM2 6L3 6L3 7L2 7L2 6Z"></path></svg></a></div><div class="Art-module__QRr6PW__row2"><div class="Art-module__QRr6PW__left"></div><div class="Art-module__QRr6PW__canvasContainer"><div class="Art-module__QRr6PW__background"><svg width="326" height="276" viewBox="0 0 326 276" xmlns="http://www.w3.org/2000/svg"><circle cx="13.5" cy="12" r="1"></circle><circle cx="13.5" cy="34.8182" r="1"></circle><circle cx="13.5" cy="57.6364" r="1"></circle><circle cx="13.5" cy="80.4545" r="1"></circle><circle cx="13.5" cy="103.273" r="1"></circle><circle cx="13.5" cy="126.091" r="1"></circle><circle cx="13.5" cy="148.909" r="1"></circle><circle cx="13.5" cy="171.727" r="1"></circle><circle cx="13.5" cy="194.545" r="1"></circle><circle cx="13.5" cy="217.364" r="1"></circle><circle cx="13.5" cy="240.182" r="1"></circle><circle cx="13.5" cy="263" r="1"></circle><circle cx="33.5" cy="12" r="1"></circle><circle cx="33.5" cy="34.8182" r="1"></circle><circle cx="33.5" cy="57.6364" r="1"></circle><circle cx="33.5" cy="80.4545" r="1"></circle><circle cx="33.5" cy="103.273" r="1"></circle><circle cx="33.5" cy="126.091" r="1"></circle><circle cx="33.5" cy="148.909" r="1"></circle><circle cx="33.5" cy="171.727" r="1"></circle><circle cx="33.5" cy="194.545" r="1"></circle><circle cx="33.5" cy="217.364" r="1"></circle><circle cx="33.5" cy="240.182" r="1"></circle><circle cx="33.5" cy="263" r="1"></circle><circle cx="53.5" cy="12" r="1"></circle><circle cx="53.5" cy="34.8182" r="1"></circle><circle cx="53.5" cy="57.6364" r="1"></circle><circle cx="53.5" cy="80.4545" r="1"></circle><circle cx="53.5" cy="103.273" r="1"></circle><circle cx="53.5" cy="126.091" r="1"></circle><circle cx="53.5" cy="148.909" r="1"></circle><circle cx="53.5" cy="171.727" r="1"></circle><circle cx="53.5" cy="194.545" r="1"></circle><circle cx="53.5" cy="217.364" r="1"></circle><circle cx="53.5" cy="240.182" r="1"></circle><circle cx="53.5" cy="263" r="1"></circle><circle cx="73.5" cy="12" r="1"></circle><circle cx="73.5" cy="34.8182" r="1"></circle><circle cx="73.5" cy="57.6364" r="1"></circle><circle cx="73.5" cy="80.4545" r="1"></circle><circle cx="73.5" cy="103.273" r="1"></circle><circle cx="73.5" cy="126.091" r="1"></circle><circle cx="73.5" cy="148.909" r="1"></circle><circle cx="73.5" cy="171.727" r="1"></circle><circle cx="73.5" cy="194.545" r="1"></circle><circle cx="73.5" cy="217.364" r="1"></circle><circle cx="73.5" cy="240.182" r="1"></circle><circle cx="73.5" cy="263" r="1"></circle><circle cx="93.5" cy="12" r="1"></circle><circle cx="93.5" cy="34.8182" r="1"></circle><circle cx="93.5" cy="57.6364" r="1"></circle><circle cx="93.5" cy="80.4545" r="1"></circle><circle cx="93.5" cy="103.273" r="1"></circle><circle cx="93.5" cy="126.091" r="1"></circle><circle cx="93.5" cy="148.909" r="1"></circle><circle cx="93.5" cy="171.727" r="1"></circle><circle cx="93.5" cy="194.545" r="1"></circle><circle cx="93.5" cy="217.364" r="1"></circle><circle cx="93.5" cy="240.182" r="1"></circle><circle cx="93.5" cy="263" r="1"></circle><circle cx="113.5" cy="12" r="1"></circle><circle cx="113.5" cy="34.8182" r="1"></circle><circle cx="113.5" cy="57.6364" r="1"></circle><circle cx="113.5" cy="80.4545" r="1"></circle><circle cx="113.5" cy="103.273" r="1"></circle><circle cx="113.5" cy="126.091" r="1"></circle><circle cx="113.5" cy="148.909" r="1"></circle><circle cx="113.5" cy="171.727" r="1"></circle><circle cx="113.5" cy="194.545" r="1"></circle><circle cx="113.5" cy="217.364" r="1"></circle><circle cx="113.5" cy="240.182" r="1"></circle><circle cx="113.5" cy="263" r="1"></circle><circle cx="133.5" cy="12" r="1"></circle><circle cx="133.5" cy="34.8182" r="1"></circle><circle cx="133.5" cy="57.6364" r="1"></circle><circle cx="133.5" cy="80.4545" r="1"></circle><circle cx="133.5" cy="103.273" r="1"></circle><circle cx="133.5" cy="126.091" r="1"></circle><circle cx="133.5" cy="148.909" r="1"></circle><circle cx="133.5" cy="171.727" r="1"></circle><circle cx="133.5" cy="194.545" r="1"></circle><circle cx="133.5" cy="217.364" r="1"></circle><circle cx="133.5" cy="240.182" r="1"></circle><circle cx="133.5" cy="263" r="1"></circle><circle cx="153.5" cy="12" r="1"></circle><circle cx="153.5" cy="34.8182" r="1"></circle><circle cx="153.5" cy="57.6364" r="1"></circle><circle cx="153.5" cy="80.4545" r="1"></circle><circle cx="153.5" cy="103.273" r="1"></circle><circle cx="153.5" cy="126.091" r="1"></circle><circle cx="153.5" cy="148.909" r="1"></circle><circle cx="153.5" cy="171.727" r="1"></circle><circle cx="153.5" cy="194.545" r="1"></circle><circle cx="153.5" cy="217.364" r="1"></circle><circle cx="153.5" cy="240.182" r="1"></circle><circle cx="153.5" cy="263" r="1"></circle><circle cx="173.5" cy="12" r="1"></circle><circle cx="173.5" cy="34.8182" r="1"></circle><circle cx="173.5" cy="57.6364" r="1"></circle><circle cx="173.5" cy="80.4545" r="1"></circle><circle cx="173.5" cy="103.273" r="1"></circle><circle cx="173.5" cy="126.091" r="1"></circle><circle cx="173.5" cy="148.909" r="1"></circle><circle cx="173.5" cy="171.727" r="1"></circle><circle cx="173.5" cy="194.545" r="1"></circle><circle cx="173.5" cy="217.364" r="1"></circle><circle cx="173.5" cy="240.182" r="1"></circle><circle cx="173.5" cy="263" r="1"></circle><circle cx="193.5" cy="12" r="1"></circle><circle cx="193.5" cy="34.8182" r="1"></circle><circle cx="193.5" cy="57.6364" r="1"></circle><circle cx="193.5" cy="80.4545" r="1"></circle><circle cx="193.5" cy="103.273" r="1"></circle><circle cx="193.5" cy="126.091" r="1"></circle><circle cx="193.5" cy="148.909" r="1"></circle><circle cx="193.5" cy="171.727" r="1"></circle><circle cx="193.5" cy="194.545" r="1"></circle><circle cx="193.5" cy="217.364" r="1"></circle><circle cx="193.5" cy="240.182" r="1"></circle><circle cx="193.5" cy="263" r="1"></circle><circle cx="213.5" cy="12" r="1"></circle><circle cx="213.5" cy="34.8182" r="1"></circle><circle cx="213.5" cy="57.6364" r="1"></circle><circle cx="213.5" cy="80.4545" r="1"></circle><circle cx="213.5" cy="103.273" r="1"></circle><circle cx="213.5" cy="126.091" r="1"></circle><circle cx="213.5" cy="148.909" r="1"></circle><circle cx="213.5" cy="171.727" r="1"></circle><circle cx="213.5" cy="194.545" r="1"></circle><circle cx="213.5" cy="217.364" r="1"></circle><circle cx="213.5" cy="240.182" r="1"></circle><circle cx="213.5" cy="263" r="1"></circle><circle cx="233.5" cy="12" r="1"></circle><circle cx="233.5" cy="34.8182" r="1"></circle><circle cx="233.5" cy="57.6364" r="1"></circle><circle cx="233.5" cy="80.4545" r="1"></circle><circle cx="233.5" cy="103.273" r="1"></circle><circle cx="233.5" cy="126.091" r="1"></circle><circle cx="233.5" cy="148.909" r="1"></circle><circle cx="233.5" cy="171.727" r="1"></circle><circle cx="233.5" cy="194.545" r="1"></circle><circle cx="233.5" cy="217.364" r="1"></circle><circle cx="233.5" cy="240.182" r="1"></circle><circle cx="233.5" cy="263" r="1"></circle><circle cx="253.5" cy="12" r="1"></circle><circle cx="253.5" cy="34.8182" r="1"></circle><circle cx="253.5" cy="57.6364" r="1"></circle><circle cx="253.5" cy="80.4545" r="1"></circle><circle cx="253.5" cy="103.273" r="1"></circle><circle cx="253.5" cy="126.091" r="1"></circle><circle cx="253.5" cy="148.909" r="1"></circle><circle cx="253.5" cy="171.727" r="1"></circle><circle cx="253.5" cy="194.545" r="1"></circle><circle cx="253.5" cy="217.364" r="1"></circle><circle cx="253.5" cy="240.182" r="1"></circle><circle cx="253.5" cy="263" r="1"></circle><circle cx="273.5" cy="12" r="1"></circle><circle cx="273.5" cy="34.8182" r="1"></circle><circle cx="273.5" cy="57.6364" r="1"></circle><circle cx="273.5" cy="80.4545" r="1"></circle><circle cx="273.5" cy="103.273" r="1"></circle><circle cx="273.5" cy="126.091" r="1"></circle><circle cx="273.5" cy="148.909" r="1"></circle><circle cx="273.5" cy="171.727" r="1"></circle><circle cx="273.5" cy="194.545" r="1"></circle><circle cx="273.5" cy="217.364" r="1"></circle><circle cx="273.5" cy="240.182" r="1"></circle><circle cx="273.5" cy="263" r="1"></circle><circle cx="293.5" cy="12" r="1"></circle><circle cx="293.5" cy="34.8182" r="1"></circle><circle cx="293.5" cy="57.6364" r="1"></circle><circle cx="293.5" cy="80.4545" r="1"></circle><circle cx="293.5" cy="103.273" r="1"></circle><circle cx="293.5" cy="126.091" r="1"></circle><circle cx="293.5" cy="148.909" r="1"></circle><circle cx="293.5" cy="171.727" r="1"></circle><circle cx="293.5" cy="194.545" r="1"></circle><circle cx="293.5" cy="217.364" r="1"></circle><circle cx="293.5" cy="240.182" r="1"></circle><circle cx="293.5" cy="263" r="1"></circle><circle cx="313.5" cy="12" r="1"></circle><circle cx="313.5" cy="34.8182" r="1"></circle><circle cx="313.5" cy="57.6364" r="1"></circle><circle cx="313.5" cy="80.4545" r="1"></circle><circle cx="313.5" cy="103.273" r="1"></circle><circle cx="313.5" cy="126.091" r="1"></circle><circle cx="313.5" cy="148.909" r="1"></circle><circle cx="313.5" cy="171.727" r="1"></circle><circle cx="313.5" cy="194.545" r="1"></circle><circle cx="313.5" cy="217.364" r="1"></circle><circle cx="313.5" cy="240.182" r="1"></circle><circle cx="313.5" cy="263" r="1"></circle></svg></div><canvas class="Art-module__QRr6PW__Canvas"></canvas><div class="Art-module__QRr6PW__tenX"><svg width="9" height="7" viewBox="0 0 9 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M0 7V0H1V1H2V2H3V3H4V4H3V5H2V6H1V7H0Z"></path><path d="M5 7V0H6V1H7V2H8V3H9V4H8V5H7V6H6V7H5Z"></path></svg><div class="text-smallcaps">10x</div></div></div><div class="Art-module__QRr6PW__right"></div></div><div class="Art-module__QRr6PW__bottom"></div></div><div class="FeaturedPost-module__8Dh16G__copy"><a class="FeaturedPost-module__8Dh16G__copyLink" target="_blank" href="/blog/meet-stripes-knowledge-ai-platform"><div class="text-md FeaturedPost-module__8Dh16G__title" data-testid="featured-post-title"><span class="FeaturedPost-module__8Dh16G__highlight">Meet Stripe's Knowledge AI Platform<span class="FeaturedPost-module__8Dh16G__arrow"><svg viewBox="0 0 6 6" fill="none" xmlns="http://www.w3.org/2000/svg"><rect y="5" width="1" height="1" fill="currentColor"></rect><rect x="1" y="4" width="1" height="1" fill="currentColor"></rect><rect x="2" y="3" width="1" height="1" fill="currentColor"></rect><rect x="3" y="2" width="1" height="1" fill="currentColor"></rect><rect x="4" y="1" width="1" height="1" fill="currentColor"></rect><rect x="5" width="1" height="6" fill="currentColor"></rect><rect width="6" height="1" fill="currentColor"></rect></svg></span></span></div><div class="text-sm FeaturedPost-module__8Dh16G__body" data-testid="featured-post-summary"><span class="FeaturedPost-module__8Dh16G__bodyHighlight">Stripe's Knowledge AI Platform is our versatile AI agent platform built to handle diverse non-coding knowledge work, from quick queries to complex,...</span><div class="FeaturedPost-module__8Dh16G__tags"><div class="text-smallcaps Tag-module__zQcxqq__tag FeaturedPost-module__8Dh16G__tagHighlight">Engineering</div><div class="text-smallcaps Tag-module__zQcxqq__tag FeaturedPost-module__8Dh16G__tagHighlight">AI</div></div></div></a></div></div></section><section class="section-module__Yglhya__section Footer-module__Grjkva__section"><div class="Footer-module__Grjkva__backgroundIcons"><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg></div><div class="Footer-module__Grjkva__docsSection"><div class="TableHeader-module__qdaoaq__label Footer-module__Grjkva__docsLabel text-smallcaps"><span>/</span>Docs</div><p class="text-sm Footer-module__Grjkva__docsDescription">Explore our guides and examples to integrate Stripe.</p><a target="_blank" class="buttonSm Footer-module__Grjkva__docsButton" href="https://docs.stripe.com">Learn more</a></div><div class="Footer-module__Grjkva__linksSection"><div class="TableHeader-module__qdaoaq__label Footer-module__Grjkva__linksLabel text-smallcaps"><span>/</span>Social</div><div class="Footer-module__Grjkva__linkList"><a href="https://www.youtube.com/stripedevelopers" class="text-smallcaps Tag-module__zQcxqq__tag" target="_blank">Youtube</a><a href="https://x.com/stripedev" class="text-smallcaps Tag-module__zQcxqq__tag" target="_blank">Twitter/X</a><a href="https://discord.com/channels/841573134531821608/841573134531821612" class="text-smallcaps Tag-module__zQcxqq__tag" target="_blank">Discord</a></div><div class="TableHeader-module__qdaoaq__label Footer-module__Grjkva__resourcesLabel text-smallcaps"><span>/</span>Resources</div><div class="Footer-module__Grjkva__linkList"><a href="https://docs.stripe.com" class="text-smallcaps Tag-module__zQcxqq__tag" target="_blank">Docs</a><a href="https://www.meetup.com/pro/stripe/" class="text-smallcaps Tag-module__zQcxqq__tag" target="_blank">Developer Meetups</a></div></div><div class="Footer-module__Grjkva__backgroundIcons Footer-module__Grjkva__backgroundIconsSecondRow"><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg></div><div class="Footer-module__Grjkva__copyright"><div class="GlobeIcon-module__AdM6KG__icon"><canvas class="GlobeIcon-module__AdM6KG__ellipseCanvas"></canvas><svg width="85" height="42" viewBox="0 0 83 42" fill="none"><circle cx="63" cy="21" r="19.5" stroke="var(--globeGraphicColor)"></circle><circle cx="19.5" cy="21" r="19.5" stroke="var(--globeGraphicColor)"></circle><path d="M3.5 9.75 35 9.75" stroke="var(--globeGraphicColor)"></path><path d="M0 21 39 21" stroke="var(--globeGraphicColor)"></path><path d="M3 32 35 32.5" stroke="var(--globeGraphicColor)"></path><path d="M54.096 16.056H57.948V16.788H58.68V17.532H59.424V24.336H58.68V25.08H57.948V25.836H54.096V16.056ZM57.816 24.948V24.216H58.56V17.664H57.816V16.92H54.984V24.948H57.816ZM60.336 16.056H65.664V16.92H61.224V20.496H64.92V21.372H61.224V24.948H65.664V25.836H60.336V16.056ZM68.076 22.116H67.332V19.152H66.576V16.056H67.464V19.02H68.196V21.984H68.94V24.216H69.552V21.984H70.296V19.02H71.04V16.056H71.904V19.152H71.16V22.116H70.428V24.336H69.684V25.836H68.808V24.336H68.076V22.116Z" fill="var(--globeGraphicColor)"></path></svg></div><span class="text-smallcaps">© <!-- -->2026<!-- --> Stripe, Inc.</span></div><div class="Footer-module__Grjkva__miscList Footer-module__Grjkva__linkList"><a href="https://stripe.com/privacy" class="text-smallcaps Tag-module__zQcxqq__tag" target="_blank">Privacy</a><a href="https://stripe.com/legal" class="text-smallcaps Tag-module__zQcxqq__tag" target="_blank">Legal</a><a href="https://stripe.com" class="text-smallcaps Tag-module__zQcxqq__tag" target="_blank">Stripe.com</a></div><div class="Footer-module__Grjkva__backgroundIcons"><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg><svg class="Footer-module__Grjkva__backgroundIcon BackgroundIcon-module__GtvU_G__icon" width="8" height="8" viewBox="0 0 8 8" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3.5 4.5V8H4.5V4.5H8V3.5H4.5V0H3.5V3.5H0V4.5H3.5Z" fill="currentColor"></path></svg></div></section><div class="EndlessFooter-module__1AI9IW__footer"><canvas class="EndlessFooter-module__1AI9IW__canvas"></canvas></div><div class="Console-module__X7eFnq__viewport"></div></div><!--$--><!--/$--></main></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"postData":{"slug":"how-stripe-uses-graph-search-and-state-machines-to-auto-remediate-a-global-database-fleet","title":"How Stripe uses graph search and state machines to auto-remediate a global database fleet","summary":"Discover how we modeled our MongoDB infrastructure as a traversable graph and then use pathfinding algorithms to dynamically compute and execute recovery plans. This automated approach reduced pager volume by 30% (~200 pages/year), eliminated 12 days of unhealthy shard states annually, and supports new shard layouts with zero manual effort.","authors":["Pragya Mehta","Sai Samant"],"date":"2026-07-16","location":[],"type":"Blog","topics":["Engineering","featured"],"artConfig":{"ampX":1.8333372565731405,"ampY":0.8067956981211901,"aspectRatio":0.5175788556730374,"axis":"x","lump":-0.13829023626074194,"count":16,"freq":0.641273811146617,"isDial":false,"isLineart":false,"isRing":false,"isSpiral":false,"left":-97,"mouseX":0.5031075581785291,"mouseY":0.4878755620848388,"noise":3.3,"scale":0.8627977433204652,"shape":"ellipse","top":-58,"twirl":0.10035621044039726,"twist":16.907588650368155,"velocity":0.7760455271862448,"isBalls":true,"isMatrix":true},"content":"\nAt 2 a.m. on a Tuesday, an on-call engineer's pager fires. A MongoDB shard—one of 2,000 that collectively processed $1.9 trillion in payments in 2025—has entered a degraded state. Two nodes are down, and a third is blocking an index build due to a misconfigured vote. To nurse the shard back to health, the engineer will need to spend 15 minutes carefully sequencing manual operations, then up to 3 hours triggering and monitoring each operation.\n\nThis scenario played out hundreds of times a year across Stripe's Document Database fleet—our global MongoDB infrastructure spanning over 40 distinct shard layouts, from cost-optimized configurations to high-availability setups designed for disaster recovery. The combinatorial explosion of failure modes made it impossible to anticipate every scenario with static, hand-coded automation. We needed a fundamentally different approach.\n\nOur solution treats infrastructure state as a traversable graph and lets a pathfinding algorithm discover recovery sequences at runtime. By applying breadth-first search, and later, Dijkstra's algorithm, over a simulated state machine, we built a system that dynamically computes and executes multi-step recovery plans. As a result, we’ve seen a 30% reduction in pager volume (~200 fewer pages per year), 12 fewer days of shards stuck in unhealthy states annually, and zero additional work required to support new shard layouts.\n\n## Why static automation breaks at scale\n\nOur original auto-remediation system (V1) operated as a single-step, hard-coded state machine. A set of plugins—each responsible for one issue, like \"fix votes,\" \"fix priority,\" or \"rebuild downed node\"—ran in a carefully ordered sequence based on hard-coded ranking and assumptions within a reconciliation loop.\n\nThis worked well for simple cases, like if a single node went down, or one or two misconfigured votes, but it had fundamental limitations:\n\n- Implicit dependencies between plugins meant that ordering was fragile. The \"fix votes\" and \"fix priorities\" plugins required no nodes to be down, but they had higher priority than the \"node down\" plugin. We could only resolve these circular dependencies with manual intervention.\n\n- Multi-failure scenarios exposed blind spots. When multiple impaired nodes combined with voting and priority misconfigurations, the sequential, single-issue-at-a-time approach couldn't find a valid ordering. The system would exhaust its options and page an operator.\n\n- Layout coupling was everywhere. Each plugin contained layout-specific safety checks, so onboarding a new layout to the V1 remediation platform required auditing the logic of every plugin and workflow, a process that took about 1 week for each new layout.\n\n- Workflow cancellations left shards stranded. A canceled workflow could leave the shard in an intermediate state that V1 never anticipated. These orphaned states fell outside any plugin's preconditions and required manual cleanup.\n\nIn one six-month window, the control plane paged operators 124 times for misconfigured shards, and 32 times for single-node-down scenarios complicated by additional health issues. Critical operations like index builds and planned maintenance were blocked for an average of one hour per incident.\n\n## Infrastructure as a traversable graph\n\n### Reframing the question\n\nThe breakthrough came when we stopped asking, \"What's the most important issue to fix right now?\", which is a local, greedy question. Instead, we asked, \"What sequence of operations transforms the current shard state into a healthy state without ever violating safety invariants?\" Once we framed it that way, we realized we were looking at a graph search problem.\n\n\n\n### Modeling a shard’s health as a state machine\n\nIn MongoDB, cluster topology is controlled by assigning nodes specific roles: voting rights determine participation in leader elections and quorum writes, priority levels control a node's eligibility to become the primary, and hidden status keeps a node out of the client read path. We capture these per-node attributes, along with shard-level properties like oplog sizes, regional distribution, and quorum status, to model a shard’s state as a node in a directed graph.\n\nEdges in the graph represent atomic infrastructure operations: adding or removing a vote or priority, hiding or unhiding a node, rebuilding a node, or resizing an oplog. These are steps we need to take on the path to getting the shard healthy again.\n\n### Finding the path with breadth-first search over simulated states\n\nThe initial state is the shard's current ground reality. The goal state, the desired, healthy configuration, is the shard's Source of Truth (SOT). Invalid states violate safety invariants. These could be an even number of votes, a minority of healthy voting nodes, which puts write availability at risk, or a node in an invalid state, like priority=1 and vote=0.\n\nWith the state space defined, we use breadth-first search (BFS) to find the shortest sequence of operations from the current state to the goal state. The algorithm generates successor states at each step, prunes any that are invalid, and returns the first path that reaches the goal. Since BFS traverses level-by-level, it guarantees that the first solution uses the minimum number of operations.\n\nConsider a healthy shard where three things go wrong simultaneously: a data node goes down, the backup node goes down, and another data node loses its vote and priority. V1 is completely stuck—the vote-fix plugin requires no nodes to be down, and the node-rebuild plugin requires votes to be correct. Our state machine, however, explores all valid orderings:\n\n```\nCurrent State → [Rebuild backup] → [Fix vote+priority] → [Rebuild data node] → Healthy\nCurrent State → [Fix vote+priority] → [Rebuild backup] → [Rebuild data node] → Healthy\nCurrent State → [Fix vote+priority] → [Rebuild data node] → [Rebuild backup] → Healthy\n```\n\nEach path is validated against invariants at every intermediate step. Fixing the vote/priority issue first is safe since it maintains quorum, which then unlocks subsequent operations.\n\n### In-memory simulation and durable execution with Temporal\n\nWe orchestrate our remediation workflows with [Temporal](https://temporal.io/), a durable execution framework that guarantees a workflow interrupted by a deploy, crash, or timeout can resume exactly where it left off—essential for node rebuilds that can take hours.\n\nBut Temporal's durability model creates a challenge for pathfinding. Our remediation building blocks are implemented as Temporal workflows and activities that are tightly coupled to Temporal's workflow.Context. They make network calls, wait for infrastructure changes, and checkpoint progress. We can't call these functions in a loop to simulate thousands of state transitions. A single simulated path through a five-step recovery would take minutes of real I/O; exploring hundreds of candidate paths would be intractable.\n\nWe solved this by introducing an abstraction layer: a CommonContext interface that can be backed by either Temporal's real workflow context, or a lightweight in-memory implementation.\n\n```go\n// CommonContext abstracts Temporal's workflow.Context, enabling both\n// production execution and local simulation.\ntype CommonContext interface {\n ExecuteActivity(opts workflow.ActivityOptions, resultPtr any, activity any, args ...any) error\n\n ExecuteLocalActivity(opts workflow.LocalActivityOptions, resultPtr any, activity any, args ...any) error\n\n ExecuteWorkflow(opts workflow.ChildWorkflowOptions, resultPtr any, workflow any, args ...any) error\n\n WithExternalDeps(sotProvider, grProvider SerializedShardProviderData, simState *SMShardState) (CommonContext, error)\n}\n```\n\nThe interface wraps Temporal's core primitives so that remediation logic can call them without knowing whether it's running inside a real workflow or a local simulation. The key method is WithExternalDeps, which swaps in simulated SOT and Ground Reality providers along with an in-memory shard state, allowing the system to explore thousands of state transitions without touching real infrastructure. In production, CommonContext delegates to Temporal and makes real API calls. In simulation, it delegates to an in-memory state provider that applies operations as pure state transformations. Both modes run the same business logic.\n\nThis gives us a critical property: what you simulate is what you execute. There’s no separate planning codebase that could diverge from the execution codebase. When the simulator determines that a sequence of operations will heal a shard, we have very high confidence that executing those same operations in production will produce the same result.\n\nThe simulation approach transforms minutes of I/O-bound execution into just milliseconds of CPU-bound computation. The BFS pathfinder can explore hundreds of state transitions per second, evaluating complete multi-step recovery paths in the time it would take to make a single network call.\n\n## Evolving the algorithm: From BFS to Dijkstra\n\nBFS finds the optimal path to a fully healthy state, but it fails entirely if no complete path exists. Consider a shard with an extra node that automation can’t reach, alongside another node that has a missing vote. BFS returns no path at all, because the full goal state is unreachable—even though it’s trivial to fix the missing vote, and leaving it unfixed puts the shard at elevated risk of losing write quorum.\n\nWhen our algorithm couldn’t reach a perfectly healthy state, could it find the least degraded state instead?\n\n### Weighted graph search\n\nWe evolved the algorithm to use Dijkstra's shortest-path algorithm over a weighted graph. The edge weight for each operation accounts for misconfiguration and estimated time:\n\n```\nedgeWeight(state, operation) = misconfiguration(state) × estimatedTime(operation)\n```\n\nmisconfiguration(state) quantifies how far the current state deviates from healthy, using a rules-based scoring system that accounts for missing votes, incorrect priorities, downed nodes, and other issues. estimatedTime(operation) estimates wall-clock duration–a vote change takes seconds, while a node rebuild takes an hour. Using the product of these two for edge weight means the algorithm minimizes time spent in misconfigured states, prioritizing quick fixes that reduce exposure before committing to longer operations.\n\n\n\n### Partial remediation\n\nBecause Dijkstra's algorithm explores paths to all reachable states rather than only the goal state, we can also get partial remediation. When no complete path exists, the algorithm returns the path to the least misconfigured state. The ephemeral-node example resolves itself: Dijkstra fixes the vote, reduces the shard's risk, and leaves the ephemeral node for operator attention.\n\n\n\n### Extensibility through composable rules\n\nThe misconfiguration scoring is designed as a set of composable rules. The core function curries the SOT and returns a closure that can rapidly score any candidate state during graph traversal:\n\n```go\ntype MisconfigRule func(*ar.SMShardState, *ar.SMShardState) int\n\nfunc GetMisconfigFunc(misconfigRules map[string]MisconfigRule, sot *ar.SMShardState) func(current *ar.SMShardState) int {\n return func(current *ar.SMShardState) int {\n misconfig := 0\n\n for _, rule := range misconfigRules {\n misconfig += rule(current, sot)\n }\n \n return misconfig\n }\n}\n```\n\nEach rule targets a specific class of misconfiguration—downed nodes, voting mismatches, priority mismatches, oplog sizing, hidden status, and availability zone imbalance—and we can add new rules independently as our understanding of severity evolves:\n\n```go\nfunc ListAllMisconfigRules() map[string]MisconfigRule {\n return map[string]MisconfigRule{\n \"DownNodeMisconfigRule\": DownNodeMisconfigRule,\n \"VotingMisconfigRule\": VotingMisconfigRule,\n \"PriorityMisconfigRule\": PriorityMisconfigRule,\n \"OplogResizeMisconfigRule\": OplogResizeMisconfigRule,\n \"HiddenStatusMisconfigRule\": HiddenStatusMisconfigRule,\n \"AZMisconfigRule\": AZMisconfigRule,\n }\n }\n```\n\nOperation costs are expressed as relative weights that reflect how dramatically operations differ in duration—a node rebuild takes roughly 33 times longer than a config change:\n\n```go\nconst (\n\n OPTIME_NODE_UPDATE_MONGO_CONFIG = 3\n OPTIME_SHARD_UPDATE_OPLOG = 19\n OPTIME_NODE_REBUILD = 101\n\n )\n```\n\nThe named rule map also makes debugging straightforward. The logs for a generated path show exactly how each rule contributed to the misconfiguration score at every intermediate state.\n\n## Impact\n\nThe state machine auto-remediation system has fundamentally changed our operational baseline. We’ve seen a 30% reduction in pager volume, or ~200 fewer pages a year, along with 12 days of fleet health reclaimed annually from shards that used to sit in degraded states until someone noticed them. Critical operations like index builds and planned maintenance now proceed without getting bottlenecked by manual stabilization.\n\nThe system is also layout-independent: when we onboarded a new layout for a planned failover, auto-remediation worked with zero code changes. And because workflows can be canceled mid-execution, the system can compute a fresh recovery path from whatever intermediate state was left behind.\n\nBeyond metrics, the system has changed how we think about failure. Instead of writing runbooks for anticipated scenarios, we define invariants and available operations, and let the algorithm compose them into recovery plans.\n\n## Future directions\n\nRight now, the state machine answers one question: \"How do I get this shard back to its declared healthy state?\" But the framework is extensible.\n\nThe most immediate extension is layout transformation. Our database fleet has over 40 shard layouts, and migrating a shard from one topology to another currently means writing a bespoke workflow that encodes every intermediate step. Instead, we can set the goal state and let the algorithm discover a safe migration sequence the same way it discovers a recovery sequence. We can use simple config updates for fleet-wide topology changes rather than needing to write bespoke migration workflows.\n\nWe can also adapt the state model to handle blue-green migrations, where a shard temporarily runs two \"colors\" of nodes during a version rollout. The pathfinder must be topology-aware, understanding which color is active and what \"healthy\" means at each phase of the migration. We want to be able to overlay migration context onto the shard's SOT, allowing the same Dijkstra-based planner to rebalance votes, rebuild failed nodes with the correct version binary, and resize oplogs. Then we can safely automate rollbacks when health signals degrade after a traffic shift.\n\nFurther out, we want the system to orchestrate planned maintenance alongside reactive healing. Right now, we execute one operation at a time on a single node across the entire shard, but when a node is already offline for a version upgrade, we could batch an oplog resize or index build into the same window. The system could discover these batching opportunities by modeling \"node offline for maintenance\" as a state where multiple operations become safe to execute together.\n\n## Conclusion\n\nIf you can model your infrastructure's state space and define valid transitions, recovery becomes a search problem. The engineering challenge is making that search practical—building a simulation layer that’s fast enough for exhaustive exploration, ensuring simulation fidelity matches production execution, and maintaining safety invariants that prevent the algorithm from proposing dangerous intermediate states.\n\nFor teams managing complex distributed infrastructure, this pattern of state machine modeling, simulation-based planning, and runtime pathfinding offers a compelling alternative to accumulating ever-more-specific runbooks. Runbooks encode known recovery procedures; a state machine discovers novel ones.\n\nSpecial thanks to Aarush Gupta, Jessica Glustien, Neeraj Joshi, Puneet Oberai, Runchen Yan, Shalin Shekhar Mangar and Srivatsan Sridharan for their contributions to this project.\n\nIf building systems like this interests you, [we're hiring](https://stripe.com/jobs/search?query=infrastructure).\n","authorBio":"","additionalResources":[{"title":"Subscribe to Stripe Developers on YouTube.","url":"https://www.youtube.com/stripedevelopers"},{"title":"Check out the docs for the in-depth developer guidance.","url":"https://docs.stripe.com/"},{"title":"Join the Stripe Discord server to chat live with other developers.","url":"https://discord.com/invite/RuJnSBXrQn"},{"title":"Join a local Stripe Developer Meetup to learn about the latest features and network with your community.","url":"https://www.meetup.com/pro/stripe/"}],"relatedArticles":[{"slug":"building-a-data-plane-from-scratch-stripes-own-high-performance-distributed-proxy","title":"Building a data plane from scratch: Stripe’s own high-performance distributed proxy","summary":"To minimize failed payments for our users, Stripe targets 99.9995% reliability. As traffic grew beyond 1.6% of global GDP, this put intense strain on the service mesh. This is the story of the infrastructure team replacing Envoy, a popular solution, with an alternative architecture.","authors":["Jack Loughran","Cara Mecozzi"],"date":"2026-08-26","location":[],"type":"Blog","topics":["Engineering","Infrastructure"],"artConfig":{"ampX":0.34922884912043817,"ampY":0.366142391782254,"aspectRatio":3.5991310963975267,"axis":"xy","lump":0.33172155507281426,"count":70,"freq":0.4,"isDial":false,"isLineart":false,"isRing":true,"isSpiral":true,"left":4,"mouseX":0.5866846039371565,"mouseY":0.13641974309831856,"noise":0.08586734730005265,"scale":6.598315318033099,"shape":"rect","top":12,"twirl":0,"twist":1.4917267701253296,"velocity":2.061707329314202}},{"slug":"meet-stripes-knowledge-ai-platform","title":"Meet Stripe's Knowledge AI Platform","summary":"Stripe's Knowledge AI Platform is our versatile AI agent platform built to handle diverse non-coding knowledge work, from quick queries to complex, multi-day projects. By connecting employees to over 1,000 internal tools and skills, it enables secure, enterprise-scale productivity across the organization.","authors":["Anna Mason","Sharadh Krishnamurthy","Anupam Upadhyay"],"date":"2026-07-30","location":[],"type":"Blog","topics":["Engineering","AI"],"artConfig":{"ampX":0.45312970344722275,"ampY":0.6594387119449675,"aspectRatio":3.640611987476796,"axis":"xy","lump":0.6013426144607366,"count":104,"freq":0.501017894551158,"isDial":false,"isLineart":false,"isRing":true,"isSpiral":true,"left":-23,"mouseX":0.5907904613740742,"mouseY":0.1359473072765395,"noise":0.3699935179296881,"scale":7.06511910136044,"shape":"rect","top":-18,"twirl":0,"twist":7.786044806666672,"velocity":2.002865631341934}}]},"authors":[{"name":"Pragya Mehta","slug":"pragya-mehta","profile":"Pragya Mehta is an engineer on the Document Database Control Plane Platform team at Stripe","posts":[]},{"name":"Sai Samant","slug":"sai-samant","profile":"Sai Samant is a technical writer at Stripe working across the engineering organization.","posts":[{"slug":"extending-stripes-network-with-stablecoins","authorBio":"","title":"Extending Stripe’s network with stablecoins","summary":"Most of the world's money still moves in batches. A wire submitted at 2pm might settle by end of day, if you made the cutoff. ACH takes one to three business days. Cross-border payments touch correspondent banks, each adding lag and a fee. Stripe has spent over a decade building the Global Payments and Treasury Network (GPTN), a programmable infrastructure for global money movement that batches transfers to reduce cost, nets opposing flows to minimize actual cash moved, and routes through the optimal path between accounts. Under the hood, it's a graph with accounts as nodes, and payment rails as edges.","authors":["Sai Samant","Maciej Makowski"],"date":"2026-06-22","location":[],"type":"Blog","topics":["Engineering","Crypto"],"artConfig":{"ampX":1.7739755672067405,"ampY":0.7976267714202404,"aspectRatio":2.0805537294752896,"axis":"y","lump":1.3576615063250066,"count":83,"freq":5.529211488872766,"isDial":false,"isLineart":false,"isRing":false,"isSpiral":false,"left":-19,"mouseX":0.41627459064293654,"mouseY":0.14836930116731673,"noise":0.1587697581667453,"scale":3.134930970534682,"shape":"rect","top":116,"twirl":0.23255087293684482,"twist":46.54475291758776,"velocity":0.8554724747482687}},{"slug":"how-we-built-stripe-credits-a-programmable-auditable-way-to-pay-your-stripe-fees","authorBio":"","title":"How we built Stripe Credits: A programmable, auditable way to pay your Stripe fees","summary":"In this post we’ll dive into how we built a virtual cashless payment method that works for prepaid and Stripe-issued credits, and cleanly integrates with our accounting, compliance, and other frameworks.","authors":["Pratik Gupta","Sai Samant"],"date":"2026-04-16","location":[],"type":"Blog","topics":["Engineering","Payments"],"artConfig":{"ampX":1.4531950035393237,"ampY":2.060997702229768,"aspectRatio":1.164219562173821,"axis":"xy","lump":1.234843716558069,"count":79,"freq":4.072019349843264,"isDial":false,"isLineart":true,"isRing":true,"isSpiral":false,"left":-59,"mouseX":0.4670889249669388,"mouseY":0.44185271733719855,"noise":0,"scale":1.949388103350997,"shape":"rect","top":150,"twirl":0,"twist":1.3939915918931365,"velocity":1.2041009591128677}}]}],"estimatedReadingTime":"12 min read"},"__N_SSG":true},"page":"/blog/[slug]","query":{"slug":"how-stripe-uses-graph-search-and-state-machines-to-auto-remediate-a-global-database-fleet"},"buildId":"CpqpGfDouU7Dr4XKHLZDI","isFallback":false,"gsp":true,"scriptLoader":[]}</script></body></html> |