Files
nexus/sreweekly/articles/337/02-slowing-down-to-speed-up-circuit-breakers-for-slack-s-ci-cd.html
2026-09-12 17:23:01 +08:00

736 lines
93 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<html lang="en-US">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, height=device-height, initial-scale=1.0, maximum-scale=1">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<link rel="profile" href="https://gmpg.org/xfn/11" />
<link rel="shortcut icon" href="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/icons/favicon.png" sizes="16x16 32x32 48x48" type="image/png">
<!-- Optanon Consent Notice start -->
<script src="https://d34u8crftukxnk.cloudfront.net/onetrust/slack.engineering/production/scripttemplates/otSDKStub.js" type="text/javascript" charset="UTF-8" data-domain-script="019872bf-9ff8-7d80-8158-f0c910bcdfad"></script>
<!-- Optanon Consent Notice end -->
<title>Slowing Down to Speed Up &#8211; Circuit Breakers for Slack&#8217;s CI/CD | Engineering at Slack</title>
<meta name="description" content="What happens when your distributed service has challenges with stampeding herds of internal requests? How do you prevent cascading failures between internal services? How might you re-architect your workflows when naive horizontal or vertical scaling reaches their respective limits? These were the challenges facing Slack engineers during their day-to-day development workflows in 2020. Multiple internal&hellip;">
<meta property="og:url" content="https://slack.engineering/circuit-breakers/">
<meta property="og:type" content="website">
<meta property="og:title" content="Slowing Down to Speed Up &#8211; Circuit Breakers for Slack&#8217;s CI/CD">
<meta property="og:description" content="What happens when your distributed service has challenges with stampeding herds of internal requests? How do you prevent cascading failures between internal services? How might you re-architect your workflows when naive horizontal or vertical scaling reaches their respective limits? These were the challenges facing Slack engineers during their day-to-day development workflows in 2020. Multiple internal&hellip;">
<meta property="og:image" content="https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg">
<meta property="og:image:width" content="6000">
<meta property="og:image:height" content="4000">
<meta name="twitter:card" content="summary_large_image">
<meta property="twitter:domain" content="slack.engineering">
<meta property="twitter:url" content="https://slack.engineering/circuit-breakers/">
<meta name="twitter:title" content="Slowing Down to Speed Up &#8211; Circuit Breakers for Slack&#8217;s CI/CD">
<meta name="twitter:description" content="What happens when your distributed service has challenges with stampeding herds of internal requests? How do you prevent cascading failures between internal services? How might you re-architect your workflows when naive horizontal or vertical scaling reaches their respective limits? These were the challenges facing Slack engineers during their day-to-day development workflows in 2020. Multiple internal&hellip;">
<meta name="twitter:image" content="https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg">
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-590Q1VHB01"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-590Q1VHB01');
</script>
<!-- End Google Analytics -->
<title>Slowing Down to Speed Up &#8211; Circuit Breakers for Slack&#8217;s CI/CD &#8211; Engineering at Slack</title>
<meta name='robots' content='max-image-preview:large' />
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://slack.engineering/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fslack.engineering%2Fcircuit-breakers%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://slack.engineering/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fslack.engineering%2Fcircuit-breakers%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='all-css-12' href='https://slack.engineering/_static/??-eJxtjEkOgkAQRS8klopGXRjP0hQlVHoMvwjh9ja61OUf3qOlNJyTSTIqYR40gRAc+6196UCwNQhhzJNx7gV7Bnb0hyqTIjpTpqDdN1H9ko0Spcneaa/uh/6s9aRpRRH25ACxapu7UFWbAGqycc/4OF5vbXs53Q/nN25bRHg=' type='text/css' media='all' />
<script type="text/javascript" src="https://slack.engineering/_static/??-eJyFi0sOgCAQQy+kjiRGdGE8i0HEQRiUT4y3FxL3rprXvsJ91sJRlBThNEkhBQhmEUdpN1SgM+/OR+FWGRodKsgPJGFS5rLqK0n/fNFYpF+ptqj8EuUnz3ZifOB9242s0y+jLDUm" ></script><link rel="https://api.w.org/" href="https://slack.engineering/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://slack.engineering/wp-json/wp/v2/posts/14817" /> <style>img#wpstats{display:none}</style>
<link rel="icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=32" sizes="32x32" />
<link rel="icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=192" sizes="192x192" />
<link rel="apple-touch-icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=180" />
<meta name="msapplication-TileImage" content="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=270" />
</head>
<body class="wp-singular post-template-default single single-post postid-14817 single-format-standard wp-embed-responsive wp-theme-tinyspeck">
<a class="ts-skip-link" href="#main">Skip to main content</a>
<div class="ts-site">
<div class="ts-site-inner">
<div class="ts-header-before"></div>
<header class="ts-header">
<div class="ts-container">
<div class="ts-header__inner ts-header__inner-desktop">
<div class="ts-header__col ts-col-left">
<div class="ts-logo">
<a class="ts-header__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
<div class="ts-header__col ts-col-center"></div>
<div class="ts-header__col ts-col-right">
<!--<a href="https://slack.engineering/articles/">Articles</a>
<a href="https://slack.engineering/about/">About</a>-->
<span class="ts-header__search-toggle" role="button" data-search-toggle="open">
<span class="ts-header__search-toggle-label">Search</span>
<svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</span>
</div>
</div>
<div class="ts-header__inner ts-header__inner-mobile">
<div class="ts-header__col ts-col-left"></div>
<div class="ts-header__col ts-col-center">
<div class="ts-logo">
<a class="ts-header__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
<div class="ts-header__col ts-col-right">
<span class="ts-header__search-toggle" role="button" data-search-toggle="open">
<span class="ts-header__search-toggle-label">Search</span>
<svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</span>
</div>
</div>
</div>
<div class="ts-search">
<div class="ts-container">
<div class="ts-search__close">
<div class="ts-search__close-inner" role="button" data-search-toggle="close">
<span class="ts-search__close-label">Close</span>
<svg class="ts-search__close-icon ts-icon ts-icon-x" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M18 6L6 18M6 6L18 18" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</div>
</div>
<div class="ts-search__wrapper">
<form role="search" method="get" class="ts-search__nav-form" action="https://slack.engineering/">
<div class="ts-search__group">
<input class="ts-search__input" type="search" required value="" name="s" placeholder="What are you looking for?">
<button class="ts-search__submit">Search</button>
</div>
</form>
</div>
</div>
</div>
</header>
<main id="main" class="ts-site-primary">
<div class="ts-site-content ts-sidebar-enabled ts-singular-thumbnail-enabled">
<div class="ts-container">
<div class="ts-main-content">
<div class="ts-content-area">
<div class="ts-entry__header ts-entry__header-large">
<div class="ts-entry__header-inner">
<div class="ts-entry__header-info">
<div class="ts-entry__post-meta-wrapper">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">August 19, 2022</div>
<div class="ts-meta-reading-time">19 min read</div> </div>
</div>
<h1 class="ts-jumbo"><span>Slowing Down to Speed Up &#8211; Circuit Breakers for Slack&#8217;s CI/CD</span></h1> <p class="ts-hero-paragraph">How Slack increased developer productivity and prevented cascading internal failures by implementing orchestration-level circuit breakers </p>
<div class="c-article__author__meta"><div class="c-article__author"><div class="c-article__author__avatar"><img src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/T060YLK5JRH-U06VADQ87E3-403b751d539d-512.jpeg"/></div><div class="c-article__author__info"><span class="c-article__author__name">Frank Chen</span><span class="c-article__author__title">Special Projects Lead</span></div></div></div> </div>
<figure class="ts-entry__post-media post-media filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg?w=1020" class="attachment-post-thumbnail size-post-thumbnail wp-post-image" alt="" decoding="async" fetchpriority="high" srcset="https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg 5184w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg?resize=640,427 640w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg?resize=768,512 768w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg?resize=1280,853 1280w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg?resize=1536,1024 1536w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg?resize=2048,1365 2048w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg?resize=380,253 380w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg?resize=800,533 800w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/florian-wehde-vZSOzoC59RU-unsplash.jpg?resize=1160,773 1160w" sizes="(max-width: 1020px) 100vw, 1020px" /> </figure>
<figcaption class="ts-entry__caption-text">Photo by Florian Wehde on Unsplash</figcaption>
</div>
</div>
<div class="ts-entry__wrap">
<div class="ts-entry__container">
<aside class="ts-widget-area ts-sidebar__area">
<div class="ts-sidebar__inner">
<!-- Search -->
<div class="widget widget_search">
<p class="ts-section-heading">Search</p>
<form role="search" method="get" class="ts-search__form" action="https://slack.engineering/">
<div class="ts-search__container">
<input required class="ts-search__input" type="search" value="" name="s" placeholder="">
<button class="ts-search__submit"><svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg></button>
</div>
</form>
</div>
<!-- Latest Posts -->
<div class="widget">
<p class="ts-section-heading">Latest Posts</p>
<div class="widget-body ts-widget-posts">
<ul>
<li class="ts-post-item">
<article class="post-17832 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-migration ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">July 14, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/">Shipyard: How We Built Slack’s Next-Generation EC2 Platform</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17854 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-automation-testing tag-developer-productivity tag-innovation ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">June 11, 2026</div> <div class="ts-meta-reading-time">12 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/agentic-testing-where-agents-fit-in-the-e2e-testing-stack/">Agentic Testing: Where Agents Fit in the E2E Testing Stack</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17812 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-backend tag-cloud-computing tag-collaboration tag-engineering tag-infrastructure tag-innovation tag-machine-learning tag-software-development ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">May 28, 2026</div> <div class="ts-meta-reading-time">17 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/">Slack AI: The Path to Multi-Cloud</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17774 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-airflow tag-aws tag-big-data tag-data-engineering tag-data-infrastructure tag-security ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">May 5, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/">From SSH to REST: A Security-Driven Modernization of Slack&#8217;s EMR Data Pipelines</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17752 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-development tag-security tag-software-engineering ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">April 13, 2026</div> <div class="ts-meta-reading-time">14 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/managing-context-in-long-run-agentic-applications/">Managing context in long-run agentic applications</a>
</h2>
</div>
</div>
</article>
</li>
</ul>
</div>
</div>
<!-- Archives -->
<div class="widget">
<p class="ts-section-heading">Archives</p>
<ul class="wp-block-archives-list wp-block-archives">
<li><a href='https://slack.engineering/2026/'>2026</a>&nbsp;(7)</li>
<li><a href='https://slack.engineering/2025/'>2025</a>&nbsp;(9)</li>
<li><a href='https://slack.engineering/2024/'>2024</a>&nbsp;(19)</li>
<li><a href='https://slack.engineering/2023/'>2023</a>&nbsp;(16)</li>
<li><a href='https://slack.engineering/2022/'>2022</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2021/'>2021</a>&nbsp;(24)</li>
<li><a href='https://slack.engineering/2020/'>2020</a>&nbsp;(26)</li>
<li><a href='https://slack.engineering/2019/'>2019</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2018/'>2018</a>&nbsp;(11)</li>
<li><a href='https://slack.engineering/2017/'>2017</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2016/'>2016</a>&nbsp;(19)</li>
</ul>
</div>
</div>
</aside>
<div class="ts-entry__content-wrap">
<div class="ts-single-post ts-wysiwyg">
<p>What happens when your distributed service has challenges with stampeding herds of internal requests? How do you prevent cascading failures between internal services? How might you re-architect your workflows when naive horizontal or vertical scaling reaches their respective limits?</p>
<p>These were the challenges facing Slack engineers during their day-to-day development workflows in 2020. Multiple internal services that engineers used were stretched to their limits, leading to cascading failures between services. Cascading failures are positive feedback loops where one part of the system fails at scale, leading to a queue in an adjacent system that results in another system failing due to scale. For several years, internal tooling and services struggled to keep up with 10% month-over-month growth in CI/CD requests from a combination of growth in 1) internal headcount and 2) complexity of services and testing. Development across Slack slowed due to these failures, leaving internal tooling and infrastructure engineers scrambling to restore service. Engineers managed to restore service in the short-term by…</p>
<ul>
<li>Scaling appliances like Github Enterprise to the largest hardware available at the time in AWS (limiting future vertical scaling).</li>
<li>Scaling one service with more nodes to handle a new peak load (only to discover that this led to failures in another service in the infrastructure).</li>
</ul>
<p>Of course, these solutions would only work until we reached a new peak load in internal services. We needed a new way to think about this problem.</p>
<p>This article describes how Slack engineers increased developer productivity by implementing orchestration-level circuit breakers throughout internal tooling at Slack. Engineers across Developer Productivity teams at Slack applied friction to defer or shed requests in Checkpoint, a <a href="https://www.sciencedirect.com/topics/computer-science/continuous-integration">Continuous Integration</a> (CI) / <a href="https://aws.amazon.com/devops/continuous-delivery/">Continuous Delivery</a> (CD) orchestration service.</p>
<h2>Complexity and scale challenges in CI/CD orchestration and Webapp</h2>
<p>Let’s go back to 2020. We saw two classes of interrelated problems: scale and complexity.</p>
<p>Early in Slack’s history, engineers built and used CI for development and CD for deploying and releasing Slack to production. Checkpoint is our internal platform to orchestrate code builds, tests, deploys and releases. Over time, Slack grew in the number of developers and feature releases, which translated to additional load in CI/CD. As additional features were released, automated tests were written to support new features.</p>
<p><a href="https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png" target="_blank" rel="noopener"><img decoding="async" class="aligncenter wp-image-14833 size-large" src="https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png?w=688" alt="Diagram of Slack Webapp clients connecting to RTS and Flannel" width="688" height="508" srcset="https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png 3838w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png?resize=640,473 640w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png?resize=768,567 768w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png?resize=1280,946 1280w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png?resize=1536,1135 1536w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png?resize=2048,1513 2048w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png?resize=380,281 380w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png?resize=800,591 800w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-slack-arch.png?resize=1160,857 1160w" sizes="(max-width: 688px) 100vw, 688px" /></a></p>
<p><em>Figure 1. Simplified diagram of Slack’s Webapp architecture. Clients connect to three different APIs in order to present what users see efficiently in real time.</em></p>
<p>These two growth vectors led to regular new peaks of load, which resulted in novel failure modes in individual services and then cascading failures (between internal services) and incidents. Each service evolves at different rates that don’t necessarily accommodate new peaks easily through horizontal or vertical scaling (examples below).</p>
<p>Slack engineers assembled in large-scale internal incidents to resolve these cascading failures. Despite the fact that they did not affect Slack’s customers, the incidents still took time away from other work, often involving multiple teams and lasting multiple days.  While they were going on, developers at Slack experienced degraded or stopped CI test executions and limited availability for CD workflows.</p>
<ul>
<li>Slack engineers and CI tests / CD workflows encountered Git errors when peak daily test counts exceeded what our Git application could serve, causing an increase in jobs in Checkpoint (async job processing) for orchestrating tests. These delays led to build-up of queues in both Checkpoint and Jenkins for executing tests. Engineers continued to do development (with limited availability of tests) and increased these queues before the initial job could be processed.</li>
<li>Git is a cornerstone of Slack’s CI and developer tools. Git scale problems are well documented with larger organizations that build abstractions (like Google’s <a href="https://dl.acm.org/doi/fullHtml/10.1145/2854146">Piper</a>) or alternate source control (like Facebook with Mercurial). In 2019, Slack internal tools adopted Git LFS to deal with large files. During this time, the Git appliance was consistently scaled up vertically. Developer growth against a large repo in Git has been a challenge that can be solved with custom source control systems like Piper or Github’s <a href="https://github.blog/2021-04-29-scaling-monorepo-maintenance">monorepo maintenance</a>.</li>
<li>Checkpoint has an internal asynchronous job queue (using self-hosted main-main MySQL and now, using AWS’s RDS Aurora) to keep state for CI/CD orchestration. This job queue and scheduler retries failed requests. The scheduler limits concurrent jobs to reduce load and reduce failed requests on the database. This limited concurrency creates lag for users of CI/CD when there are too many jobs in a queue (like test request jobs), causing them to request the same job repeatedly which triggers a positive feedback loop and an even larger queue.</li>
<li>Slack’s internal tooling engineers regularly increased the number of test executors and environments before this project to accommodate an increasing number of engineers. Without noticing the breaking point, the combination of scale and requests from tests (i.e. test executors) and Slack environments (i.e. code under test) led to more requests than the search cluster in CI could handle gracefully, which then introduced errors and, of course, even more increasing load.</li>
</ul>
<p><a href="https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png" target="_blank" rel="noopener"><img decoding="async" class="aligncenter wp-image-14832 size-large" src="https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png?w=688" alt="Example of workflows that have the potential to cascade failures between CI services and tools" width="688" height="200" srcset="https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png 3686w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png?resize=640,186 640w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png?resize=768,223 768w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png?resize=1280,372 1280w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png?resize=1536,447 1536w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png?resize=2048,596 2048w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png?resize=380,111 380w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png?resize=800,233 800w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breakers.png?resize=1160,337 1160w" sizes="(max-width: 688px) 100vw, 688px" /></a></p>
<p><em>Figure 2. Example of workflows that have the potential to cascade failures between CI services and tools.</em></p>
<h3>Why complexity matters</h3>
<p>Slack uses integration and end-to-end tests to verify correctness for complex workflows that overlap multiple services. While Slack started with a single service (Webapp), today’s Slack involves multiple services that support the user experience. Slack clients connect to three different APIs in order to present what users see in real time (see a simplified view of the architecture in Figure 1). We describe many of the Webapp’s testing challenges and how we changed developer workflows in 2021 to mitigate many challenges resulting from <a href="https://slack.engineering/balancing-safety-and-velocity-in-ci-cd-at-slack/">large numbers of end-to-end tests</a>. Slack is a complex application with a number of configurations (e.g. team, enterprise, and cross enterprise messages). To test complex code paths, product and test engineers focused on writing expressive automated tests, which rely on a lot of moving parts (see Figure 2).</p>
<h2>Circuit breakers</h2>
<p>Software circuit breakers, a <a href="https://en.wikipedia.org/wiki/Circuit_breaker">concept borrowed from systems engineering</a>, detect faults in external systems and stop sending calls against the known faulty systems. Clients are the <a href="https://martinfowler.com/bliki/CircuitBreaker.html">typical place to implement a circuit breaker</a>. Since our CI/CD orchestration layer regulates the flow of requests through the system, our teams aligned on implementing circuit breaker clients, with multiple concurrent jobs calling the client, in the orchestrator consumer before sending requests to the next system. And of course, engineers and service owners need to be kept in the loop of the system changing behavior.</p>
<p><a href="https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breaker-state.png" target="_blank" rel="noopener"><img loading="lazy" decoding="async" class="aligncenter wp-image-14842 size-medium" src="https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breaker-state.png?w=450" alt="Circuit breaker control flow diagram with closed, open, and half-open" width="450" height="406" srcset="https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breaker-state.png 450w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/diagram-circuit-breaker-state.png?resize=380,343 380w" sizes="auto, (max-width: 450px) 100vw, 450px" /></a></p>
<p><em>Figure 3. </em><a href="https://martinfowler.com/bliki/CircuitBreaker.html"><em>Circuit breaker</em></a><em> control flow diagram.</em></p>
<p>We had a hypothesis that circuit breakers could minimize cascading failures and provide high leverage for programmatic metric queries for multiple services, instead of individual client- or service-based approaches. Unlike traditional circuit breakers in individual services, circuit breakers at the orchestration-level system could regulate the interface of requests between systems.</p>
<p>Circuit breakers opened when dependent services were encountering increased load or showing errors due to increased load. Checkpoint programmatically retrieved health metrics from multiple dependent services. Requests would then be deferred or dropped if downstream systems could not serve those requests. Circuit breakers would then close when the dependent service showed recovery, and those deferred requests would start executing once again. This management of known-faulty requests reduced cascading failure incidents that affected our company’s ability to build, test, deploy and release code as well as reduced flaky executions in CI.</p>
<h2>Methodology</h2>
<p>We started with an abstract class, implemented in <a href="https://slack.engineering/hacklang-at-slack-a-better-php/">Hacklang</a>, to ground discussions and prototype this new workflow. Our discussions focused on Checkpoint—the orchestration service—instead of build or test clients. Checkpoint orchestrates CI/CD workflows and its background job system represents the lifeblood of builds, tests, deployments, and releases at Slack. Checkpoint has an API endpoint that receives a GitHub webhook when a new commit is created. From this single commit, Checkpoint enqueues multiple background jobs that trigger Jenkins asset builds or tests and then updates test results in the database.</p>
<p>We chose to focus on circuit breaking with deferral and load shedding in Checkpoint background jobs. While circuit breakers can live in client logic (e.g. to wait for recovery or <a href="https://microservices.io/patterns/reliability/circuit-breaker.html">block to hold work</a>), Checkpoint’s background job system presented a unique opportunity because it’s the intermediary between multiple systems through the scheduler.</p>
<ul>
<li>We use <a href="https://corporate.comcast.com/stories/announcing-trickster-an-open-source-dashboard-accelerator-for-prometheus">Trickster</a> to perform programmatic look-ups for dependent service metrics in several Prometheus clusters using PromQL. This service is a frontend, proxy, and cache for queries to multiple Prometheus clusters.</li>
<li>Checkpoint didn’t need a half-open state due to internal background job retries and use of deferred CI requests. Half-open states are important for <a href="https://blog.devgenius.io/circuit-breakers-in-microservices-625654df0830">individual client requests to trickle through and to indicate recovery for these clients</a>. Since Checkpoint’s background job system has retries and this circuit breaker contains a TTL on the Prometheus query, Checkpoint readily resumes work once an open circuit breaker recovers.</li>
</ul>
<p>&nbsp;</p>
<pre><code class="language-php">namespace Checkpoint\CircuitBreaker;
use type Slack\Checkpoint\PromClient;
/*
* Generic interface for Circuit Breakers in Checkpoint.
* Downstream actions include deferral mechanisms or load shedding.
* @see https://martinfowler.com/bliki/CircuitBreaker.html
*/
enum CircuitBreakerState: string {
CLOSED = &#039;closed&#039;;
OPEN = &#039;open&#039;;
}
abstract class CircuitBreaker {
/**
* Get the state of this circuit breaker. Note the return value is intentionally
* not a `Result&lt;CircuitBreakerState&gt;`. In the case of internal errors, this must
* decide if the breaker fails open/closed.
*/
abstract protected function getState(): CircuitBreakerState;
/**
* Allow for bypassing a circuit breaker. Used as a circuit breaker for circuit breakers.
* In a subsequent class, add the following to always allow the request to pass through
* &lt;&lt;__Override, __Memoize&gt;&gt;
* public function bypass(): bool { return true; }
*/
public function bypass(): bool {
return false;
}
public function allowRequest(): bool {
$state = $this-&gt;getState();
PromClient::circuit_breaker_requests()-&gt;inc(1, darray[
&#039;breaker_type&#039; =&gt; (string)static::class,
&#039;breaker_state&#039; =&gt; (string)$state,
]);
if ($this-&gt;bypass()) return true;
return $state === CircuitBreakerState::CLOSED;
}
}
</code></pre>
<p><em>Figure 4. Simplified code for the <code>CircuitBreaker</code> class.</em></p>
<p><span style="color: var(--text--primary__color);font-family: var(--headings__FontFamily);font-size: var(--h2__FontSize);font-weight: 600">Multiple implementations</span></p>
<p>In the first implementation sprint, we focused on circuit breakers for orchestration service health:</p>
<ul>
<li>Defer test jobs when Checkpoint and Jenkins queues reach a certain threshold.</li>
<li>Defer test jobs on end-to-end test jobs when all Slack test environments are busy.</li>
<li>Load shed test executions for older commits on a branch.</li>
<li>Load shed test retries for any suite that has had consistent failures.</li>
</ul>
<p>In the next implementation sprint, we focused on circuit breakers for shared dependent services:</p>
<ul>
<li>Flannel &#8211; <a href="https://slack.engineering/flannel-an-application-level-edge-cache-to-make-slack-scale/">an edge cache</a> in multiple regions worldwide that returns frequently fetched teamwide data.</li>
<li>Vitess &#8211; the <a href="https://slack.engineering/scaling-datastores-at-slack-with-vitess/">source of truth for all customer data</a> (with MySQL syntax). Vitess is a <a href="https://vitess.io/docs/13.0/concepts/">database solution</a> for deploying, scaling, and managing large clusters of database instances.</li>
<li>Search &#8211; a <a href="https://www.youtube.com/watch?v=EQ336PTZfhU">service</a> providing indices of messages, files and people that computes a live collection (fed real-time through the job queue) and weekly collection (computed offline with information from the beginning of time).</li>
</ul>
<p>A simplified implementation is shared in the figure below for Flannel. You’ll notice a lookup in cache (along with TTL), a Prometheus range query, user messaging, and calls to Trickster using that Prometheus range query. Safety is important here—if Trickster / Prometheus clusters return an error we keep the circuit breaker closed to allow for requests to flow through. Similarly, we cache responses for consistent client requests between asynchronous jobs.</p>
<pre><code class="language-php">namespace Checkpoint\CIBot\CircuitBreaker;
use namespace Checkpoint\{CIIssue, Trickster};
use type Checkpoint\CIBot\Delta\{DeltaAnomalyType, DeltaDimensionType};
use type Checkpoint\CIIssue\ServiceDepCircuitBreakerType;
use type Checkpoint\CircuitBreaker\{Cacheable, CircuitBreaker, CircuitBreakerState};
use type Slack\Checkpoint\PromClient;
type flannel_callback_error_rate_cache_t = shape(
&#039;ts&#039; =&gt; int,
&#039;error_rate&#039; =&gt; int,
);
final class FlannelServiceDepCircuitBreaker extends CircuitBreaker {
use Cacheable;
const int TTL = 60; // Time-to-Live for cached value
const int FLANNEL_CALLBACK_ERROR_RATE_THRESHOLD = 5;
const string PROM_FLANNEL_CLUSTER = &#039;flannel&#039;;
const string PROM_FLANNEL_QUERY_GLOBAL = &#039;sum(dc:cb_errors:irate1m{error!~&quot;org_login_required&quot;})&#039;;
const string ISSUE_MESSAGE_OPEN = &#039;:warning: :broken_yellow_heart: Flannel Circuit Breaker is open. Tests are deferred&#039;;
const string ISSUE_MESSAGE_CLOSE = &#039;This circuit breaker is closed. Tests are starting again&#039;;
const string ISSUE_KEY = ServiceDepCircuitBreakerType::FLANNEL;
public function __construct(private ?\github_repos_t $repo = null, private ?\TSlackjsonValidatorPropertiesCheckpointPropertiesTestsItems $test = null) {}
&lt;&lt;__Override, __Memoize&gt;&gt;
public function getState(): CircuitBreakerState {
$cached_key = $this-&gt;getCacheKey(self::class, &#039;flannel_callback_errors&#039;);
$cached_data = \cache_get($cached_key);
$existing_error_rate = 0;
// If the cache exists, and is fresh enough, use it. Default to Closed
$result = \type_assert_type($cached_data, flannel_callback_error_rate_cache_t::class);
if ($result-&gt;is_error()) { return CircuitBreakerState::CLOSED; }
$data = $result-&gt;get();
$existing_error_rate = $data[&#039;error_rate&#039;];
if ($this-&gt;isValidCache($data[&#039;ts&#039;], static::TTL)) {
if ($existing_error_rate &lt; static::FLANNEL_CALLBACK_ERROR_RATE_THRESHOLD) {
return CircuitBreakerState::CLOSED;
} else {
return CircuitBreakerState::OPEN;
}
}
// Lets fetch the current error rate (and compare against the former one)
$result = $this-&gt;getFlannelCallbackErrorRate();
if ($result-&gt;is_error()) {
return CircuitBreakerState::CLOSED;
}
$error_rate = $result-&gt;get();
$cached_value = shape(&#039;ts&#039; =&gt; \time(), &#039;error_rate&#039; =&gt; $error_rate);
\cache_set($cached_key, $cached_value);
if ($error_rate &gt;= static::FLANNEL_CALLBACK_ERROR_RATE_THRESHOLD) {
PromClient::cibot_service_dependency_error_rate_above_threshold()-&gt;inc(1, darray[
&#039;breaker_type&#039; =&gt; (string)static::class,
]);
if ($existing_error_rate &lt; static::FLANNEL_CALLBACK_ERROR_RATE_THRESHOLD) {
CIIssue\send(static::ISSUE_MESSAGE_OPEN, DeltaDimensionType::CIRCUIT_BREAKER, DeltaAnomalyType::CIRCUIT_BREAKER_OPEN, static::ISSUE_KEY);
}
return CircuitBreakerState::OPEN;
}
// If our circuit breaker was previously open (and now closed), track this new state and mark it in our issues dataset
if ($existing_error_rate &gt;= static::FLANNEL_CALLBACK_ERROR_RATE_THRESHOLD) {
CIIssue\end(static::ISSUE_MESSAGE_CLOSE, DeltaDimensionType::CIRCUIT_BREAKER, DeltaAnomalyType::CIRCUIT_BREAKER_OPEN, static::ISSUE_KEY);
}
return CircuitBreakerState::CLOSED;
}
</code></pre>
<p><em>Figure 5. Simplified code for the <code>FlannelServiceDepCircuitBreaker</code> class.</em></p>
<h2>User interaction</h2>
<p>Each of these circuit breakers would fetch data and alert a channel when it detected an issue. The circuit breaker opening would show a few different views of the same issue. A typical workflow might involve a member of our team noticing the circuit breaker open, then escalating to the corresponding team channel with the details.</p>
<p><a href="https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-search-circuit-breaker-1-fuzzed.png" target="_blank" rel="noopener"><img loading="lazy" decoding="async" class="aligncenter wp-image-14835 size-large" src="https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-search-circuit-breaker-1-fuzzed.png?w=688" alt="Screenshot of an automated circuit breaker message in #alerts-ci-issue leading to escalation with the search team for a spike in error rate" width="688" height="300" srcset="https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-search-circuit-breaker-1-fuzzed.png 1102w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-search-circuit-breaker-1-fuzzed.png?resize=640,279 640w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-search-circuit-breaker-1-fuzzed.png?resize=768,335 768w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-search-circuit-breaker-1-fuzzed.png?resize=380,166 380w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-search-circuit-breaker-1-fuzzed.png?resize=800,348 800w" sizes="auto, (max-width: 688px) 100vw, 688px" /></a></p>
<p><em>Figure 6. Screenshot </em><em>of an automated circuit breaker message in #alerts-ci-issue leading to escalation with the search team for a spike in error rate.</em></p>
<p>In this automated circuit breaker message, each link shows different views of the same issue. Similar deferred messages are shown for the Checkpoint frontend.</p>
<p><a href="https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png" target="_blank" rel="noopener"><img loading="lazy" decoding="async" class="aligncenter wp-image-14848 size-large" src="https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png?w=688" alt="Screenshot of an automated circuit breaker message showing a service issue and test status (“Jenkins queue is currently high, Tests will continue when the queue goes down”) in Checkpoint’s PR / test view." width="688" height="76" srcset="https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png 2148w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png?resize=640,71 640w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png?resize=768,85 768w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png?resize=1280,142 1280w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png?resize=1536,170 1536w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png?resize=2048,227 2048w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png?resize=380,42 380w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png?resize=800,89 800w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/screenshot-checkpoint-blocked.png?resize=1160,129 1160w" sizes="auto, (max-width: 688px) 100vw, 688px" /></a></p>
<p><em>Figure 7. Screenshot of an automated circuit breaker message showing a service issue and test status (“Jenkins queue is currently high, Tests will continue when the queue goes down”) in Checkpoint’s PR / test view.</em></p>
<p>We mentioned earlier that Checkpoint makes queries for different service error rates, and we created a small internal library to report open circuit breakers to Slack. Measuring these specific issues (instead of seeing undifferentiated spikes of errors) enhanced our ability to reason about open circuit breakers over time. Furthermore, we extended this issue library for anomaly detection (e.g. higher than expected failure, error rate, durations, or flakiness rates) across test executors, test environments, and test suites. These in turn feed into a smoother developer experience.</p>
<p><a href="https://slack.engineering/?attachment_id=14834" target="_blank" rel="noopener"><img loading="lazy" decoding="async" class="aligncenter wp-image-14834 size-large" src="https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-backend-tests.png?w=688" alt="Screenshot of anomaly detection for test suite execution." width="688" height="318" srcset="https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-backend-tests.png 1102w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-backend-tests.png?resize=640,296 640w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-backend-tests.png?resize=768,355 768w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-backend-tests.png?resize=380,176 380w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/slack-backend-tests.png?resize=800,370 800w" sizes="auto, (max-width: 688px) 100vw, 688px" /></a></p>
<p><em>Figure 8. Screenshot of anomaly detection for test suite execution.</em></p>
<h2>Developer impact</h2>
<p>Since two sets of infrastructure and dependent service breakers were introduced, we have reduced the surface area for cascading failures (by test deferral) and smoothed out the throughput for test executions (by load shedding).</p>
<p>The result has been a significantly improved developer experience—<strong>zero cascading failure incidents in internal tooling over the last two years</strong>—and significantly reduced load for critical services that benefited the CI/CD user experience.</p>
<p>Those incidents were common before the two periods of implementation in 2020. We regularly encounter new peak loads by programmatically looking up dependent service load in CI orchestration. During a recent Git LFS incident, while the symptoms were similar to earlier incidents, the situation was localized to test executors and teams were able to fix and isolate the failures without cascading failures.</p>
<p><strong>Better feedback loops</strong>. Engineers now get feedback from Checkpoint’s frontend and via Slack when their tests are deferred until the system recovers. Before circuit breakers, these tests would flake or fail due to some downstream system being overloaded. Deferring tests overall led to fewer flakes and test executions that were less relevant. Below is an illustration from an earlier <a href="https://slack.engineering/infrastructure-observability-for-changing-the-spend-curve/">blog post</a> showing the drastic change in test requests that are no longer relevant to engineers who originally submitted them (e.g. a newer commit) and deferral of valid test requests that would have required multiple tests to resolve flakes. Note the two curve changes after each implementation period (March 2020 and August 2020).</p>
<p><a href="https://slack.engineering/wp-content/uploads/sites/7/2022/08/Picture1.png" target="_blank" rel="noopener"><img loading="lazy" decoding="async" class="aligncenter wp-image-14849 size-medium" src="https://slack.engineering/wp-content/uploads/sites/7/2022/08/Picture1.png?w=640" alt="A projection (in red) of executed test suites based on 10% growth, and curve change (in yellow) from loading shed + deferred jobs and pipeline changes." width="640" height="244" srcset="https://slack.engineering/wp-content/uploads/sites/7/2022/08/Picture1.png 1080w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/Picture1.png?resize=640,244 640w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/Picture1.png?resize=768,293 768w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/Picture1.png?resize=380,145 380w, https://slack.engineering/wp-content/uploads/sites/7/2022/08/Picture1.png?resize=800,305 800w" sizes="auto, (max-width: 640px) 100vw, 640px" /></a></p>
<p><em>Figure 9. A projection (in red) of executed test suites based on 10% growth, and curve change (in yellow) from loading shed + deferred jobs and </em><a href="https://slack.engineering/balancing-safety-and-velocity-in-ci-cd-at-slack/"><em>pipeline changes</em></a><em>.</em></p>
<p>As a side benefit, our team was able to recalibrate spend and provisioning for a portion of the fleet dedicated to running tests against commit SHAs that were not the tip of the developer branch. For details on a cumulative 10x decrease in spend compared to baseline growth, read our <a href="https://slack.engineering/infrastructure-observability-for-changing-the-spend-curve/">blog post on infrastructure observability</a>.</p>
<p>Finally, to understand feedback loops on tests, teams working across CI had aligned a business metric “time to test results.” This metric looks at the lag for developers to get results from builds and tests executing in CI. Teammates expressed concern that adding circuit breakers to defer or to shed load felt counterintuitive to sending results back quickly. Instead of seeing this metric go in the wrong direction (be slower), this metric has been steady over the last years, since many of those same tests would have failed and then presented flaky results to customers.</p>
<h3>Aside: How to frame this solution for your organization?</h3>
<p>Every organization has a <a href="https://twitter.com/williebsweet/status/1549133411272589312?s=21&amp;t=StNzwgMfKAa64NuMg0GROg">unique composition</a> of CI/CD culture, tooling, and infrastructure. While this solution can’t be copy-pasted in your organization, you can borrow ideas for separating concerns using orchestration-layer circuit breakers.</p>
<p>How did our organization decide on using circuit breakers for CI/CD? We focus on Webapp as its the main repository where business logic lives for Slack’s customers and most developers spend most of their time.</p>
<p>Test environments are complex—unit and integration tests can mean <a href="https://martinfowler.com/articles/2021-test-shapes.html">different things across organizations</a>. For example, unit and <a href="https://martinfowler.com/articles/practical-test-pyramid.html">integration tests</a> might have mocked environments and payload requests at one organization while another might rely on shared environments.</p>
<p>For Slack, many of the Webapp tests rely upon both synchronous and asynchronous requests (e.g. background jobs or websocket connections). A test for posting a message might first hit an API endpoint that then enqueues multiple asynchronous jobs to send notifications to other clients, enqueue this message for search, etc. Today, we have defined a testing taxonomy to align with Google’s <a href="https://testing.googleblog.com/2010/12/test-sizes.html">small / medium / large taxonomy</a> and emphasized shifting from an hourglass figure of mostly end-to-end and unit tests toward more medium (integration) tests to verify safety.</p>
<p>How might your organization use this pattern for CI/CD? You have a number of choices of CI/CD orchestration platforms like Gitlab, ArgoCD, GitHub Actions, or Jenkins and you can refer to the design decisions we shared in the Methodology section.</p>
<p>In choosing the right lever, we recommend using metrics, events, logs, or traces to drill down into areas of concern or common problems in the pipeline. Our approach was to initially share awareness of an open circuit breaker (see Figure 5 of a service displaying a high error rate) to Slack before opening the circuit and deferring or shedding requests. This allowed us to gain confidence on the problem space as we iteratively shipped circuit breakers in the orchestration platform.</p>
<p>Since Slack’s service layout for Webapp follows a generally complex layout, product and testing followed that pattern to create tests that flowed from this layout. Today, Slack engineers are also actively working toward a continuous deployment process. For years, Slack relied on a thin set of tests during the <a href="https://slack.engineering/deploys-at-slack/">continuous delivery process</a>. Building out an automated deployment and release testing process takes alignment due to the large number of moving pieces (e.g. when an automated rollback is requested and a fault is detected mid-stream).</p>
<p>To summarize where we’re going—we are working toward more medium-sized tests and continuous deployments, as well as automated rollbacks later in the pipeline, to help make Slack engineers more productive.</p>
<h2><strong>Conclusion</strong></h2>
<p>This article shares the decision points and outcomes of orchestration-level circuit breakers for Checkpoint, Slack’s internal CI/CD orchestration system.</p>
<p>Before this project, engineers at Slack saw challenges as requests in internal tooling reached new peaks and a system may fail and could cascade failures into other systems. These circuit breakers on the interface between systems in CI minimized cascading failures and provided high leverage for programmatic metric queries for multiple services instead of individual client or service based approaches.</p>
<p>Since the project was completed in 2020, engineers no longer encounter cascading failures between systems in internal tooling. Engineers also saw increased service availability, overall throughput from Checkpoint and fewer bad developer experiences like flakiness from failing services. The result of circuit breakers is a substantive impact on the productivity of engineers throughout Slack.</p>
<p>Multiple teams are now looking to use this framework of programmatic metric queries to move Slack toward continuous deployments—by automating builds, tests, deploys, releases, and rollbacks.</p>
<p>Are these types of sociotechnical system challenges interesting for you? If so, join us! Find out more in our <a href="https://slack.com/careers">careers page</a>.</p>
<h2>Acknowledgements</h2>
<p>This work could not have happened without help across our organization. I want to say a special thank you to <a href="https://www.linkedin.com/in/krismerrill/">Kris Merrill</a>, <a href="https://twitter.com/nolancaudill">Nolan Caudill</a>, <a href="https://www.linkedin.com/in/traviscrawford/">Travis Crawford</a>, <a href="http://www.linkedin.com/in/sylvestor88">Sylvestor George</a>, <a href="https://www.linkedin.com/in/marcellino-ornelas/">Marcellino Ornelas</a>, <a href="https://www.linkedin.com/in/hannahbottalla/">Hannah Bottalla</a>, <a href="https://www.linkedin.com/in/karenx/">Karen Xu</a>, <a href="https://github.com/doeg">Sara Bee</a>, <a href="https://www.linkedin.com/in/carlosvr98/">Carlos Valdez</a>, <a href="https://twitter.com/kogent">Mark Carey</a>, <a href="https://www.linkedin.com/in/torypayne/">Tory Payne</a>, <a href="https://www.linkedin.com/in/venkattv">Venkat Venkatanaranappa</a>, <a href="https://www.linkedin.com/in/brian-ramos-6ba41713">Brian Ramos</a>, <a href="https://www.linkedin.com/in/johnmgallagher/">John Gallagher</a> for their technical leadership and execution on this series of projects.</p>
<p>This focus on resiliency engineering was inspired by an opportunity to learn with the Resiliency Learning Lab (Lund University), organized by Nora Jones, David Woods, John Allspaw, Richard Cook, and Laura Maguire. For further reading, check out this excellent set of resources on <a href="https://www.jeli.io/howie-the-post-incident-guide/">learning from incidents</a> by Jeli.io.</p>
</div>
<div class="ts-tags-section"><span class="ts-tag">#<a href="https://slack.engineering/tags/ci-cd/" rel="tag">ci-cd</a></span><span class="ts-tag">#<a href="https://slack.engineering/tags/developer-productivity/" rel="tag">developer-productivity</a></span><span class="ts-tag">#<a href="https://slack.engineering/tags/incident-management/" rel="tag">incident-management</a></span><span class="ts-tag">#<a href="https://slack.engineering/tags/infrastructure/" rel="tag">infrastructure</a></span></div> </div>
</div>
</div>
</div>
<aside class="ts-metabar__area">
<div class="ts-metabar__inner">
<div class="ts-metabar__item">
<div class="ts-share-buttons-wrap">
<div class="ts-share-buttons-items">
<div class="ts-share-buttons-item ts-share-buttons-twitter">
<a href="https://x.com/intent/post?url=https%3A%2F%2Fslack.engineering%2Fcircuit-breakers%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-twitter" width="21" viewBox="0 0 72 65" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M22.6891 0H0L26.9014 35.3477L1.72089 65H13.355L32.3999 42.5731L49.3109 64.794H72L44.317 28.4191L44.366 28.4818L68.2015 0.412927H56.5674L38.8667 21.2573L22.6891 0ZM12.524 6.19052H19.5874L59.476 58.603H52.4126L12.524 6.19052Z" fill="currentColor"/></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-linkedin">
<a href="https://www.linkedin.com/shareArticle?mini=true&#038;url=https%3A%2F%2Fslack.engineering%2Fcircuit-breakers%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-linkedin" width="21" height="21" viewBox="0 0 70 70" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M15.8066 23.0188H1.17969V69.663H15.8066V23.0188Z" fill="currentColor"/><path d="M55.3732 22.0077C54.834 21.9403 54.2611 21.9066 53.6881 21.8729C45.4984 21.5359 40.8812 26.389 39.2635 28.4786C38.8254 29.0515 38.6231 29.3886 38.6231 29.3886V23.1536H24.6366V69.7978H38.6231H39.2635C39.2635 65.0458 39.2635 60.3274 39.2635 55.5754C39.2635 53.014 39.2635 50.4526 39.2635 47.8912C39.2635 44.7232 39.0276 41.3529 40.6116 38.4545C41.9597 36.0279 44.3862 34.8147 47.1161 34.8147C55.2047 34.8147 55.3732 42.1281 55.3732 42.8021C55.3732 42.8358 55.3732 42.8695 55.3732 42.8695V70H70.0001V39.5667C70.0001 29.1526 64.7088 23.0188 55.3732 22.0077Z" fill="currentColor"/><path d="M8.49301 16.986C13.1836 16.986 16.9861 13.1836 16.9861 8.49303C16.9861 3.80246 13.1836 0 8.49301 0C3.80244 0 0 3.80246 0 8.49303C0 13.1836 3.80244 16.986 8.49301 16.986Z" fill="currentColor"/></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-facebook">
<a href="https://www.facebook.com/sharer/sharer.php?u=https%3A%2F%2Fslack.engineering%2Fcircuit-breakers%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-facebook" width="21" height="21" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><g clip-path="url(#clip0_11_112)"><path d="M12 0C18.6274 0 24 5.37259 24 12C24 18.1352 19.3955 23.1944 13.4538 23.9121V15.667L16.7001 15.667L17.3734 12H13.4538V10.7031C13.4538 9.73417 13.6439 9.06339 14.0799 8.63483C14.5159 8.20627 15.1979 8.01993 16.1817 8.01993C16.4307 8.01993 16.6599 8.02241 16.8633 8.02736C17.1591 8.03456 17.4002 8.047 17.568 8.06467V4.74048C17.501 4.72184 17.4218 4.70321 17.3331 4.68486C17.1321 4.6433 16.8822 4.60324 16.6136 4.56806C16.0523 4.49453 15.4093 4.4423 14.9594 4.4423C13.1424 4.4423 11.7692 4.83102 10.8107 5.63619C9.65388 6.60791 9.10108 8.18622 9.10108 10.4199V12H6.62659V15.667H9.10108V23.6466C3.87432 22.3498 0 17.6277 0 12C0 5.37259 5.37259 0 12 0Z" fill="currentColor"/></g><defs><clipPath id="clip0_11_112"><rect width="24" height="24" fill="white"/></clipPath></defs></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-threads">
<a href="https://www.threads.net/intent/post?text=+https%3A%2F%2Fslack.engineering%2Fcircuit-breakers%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon" width="21" viewBox="0 0 448 512" xmlns="http://www.w3.org/2000/svg"><path d="M331.5 235.7c2.2 .9 4.2 1.9 6.3 2.8c29.2 14.1 50.6 35.2 61.8 61.4c15.7 36.5 17.2 95.8-30.3 143.2c-36.2 36.2-80.3 52.5-142.6 53h-.3c-70.2-.5-124.1-24.1-160.4-70.2c-32.3-41-48.9-98.1-49.5-169.6V256v-.2C17 184.3 33.6 127.2 65.9 86.2C102.2 40.1 156.2 16.5 226.4 16h.3c70.3 .5 124.9 24 162.3 69.9c18.4 22.7 32 50 40.6 81.7l-40.4 10.8c-7.1-25.8-17.8-47.8-32.2-65.4c-29.2-35.8-73-54.2-130.5-54.6c-57 .5-100.1 18.8-128.2 54.4C72.1 146.1 58.5 194.3 58 256c.5 61.7 14.1 109.9 40.3 143.3c28 35.6 71.2 53.9 128.2 54.4c51.4-.4 85.4-12.6 113.7-40.9c32.3-32.2 31.7-71.8 21.4-95.9c-6.1-14.2-17.1-26-31.9-34.9c-3.7 26.9-11.8 48.3-24.7 64.8c-17.1 21.8-41.4 33.6-72.7 35.3c-23.6 1.3-46.3-4.4-63.9-16c-20.8-13.8-33-34.8-34.3-59.3c-2.5-48.3 35.7-83 95.2-86.4c21.1-1.2 40.9-.3 59.2 2.8c-2.4-14.8-7.3-26.6-14.6-35.2c-10-11.7-25.6-17.7-46.2-17.8H227c-16.6 0-39 4.6-53.3 26.3l-34.4-23.6c19.2-29.1 50.3-45.1 87.8-45.1h.8c62.6 .4 99.9 39.5 103.7 107.7l-.2 .2zm-156 68.8c1.3 25.1 28.4 36.8 54.6 35.3c25.6-1.4 54.6-11.4 59.5-73.2c-13.2-2.9-27.8-4.4-43.4-4.4c-4.8 0-9.6 .1-14.4 .4c-42.9 2.4-57.2 23.2-56.2 41.8l-.1 .1z"></path></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-copy">
<a href="#" class="ts-share-buttons-link" data-share-link title="Copy link">
<svg width="21" height="21" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M10 13C10.4295 13.5741 10.9774 14.0491 11.6066 14.3929C12.2357 14.7367 12.9315 14.9411 13.6467 14.9923C14.3618 15.0435 15.0796 14.9403 15.7513 14.6897C16.4231 14.4392 17.0331 14.047 17.54 13.54L20.54 10.54C21.4508 9.59695 21.9548 8.33394 21.9434 7.02296C21.932 5.71198 21.4061 4.45791 20.4791 3.53087C19.5521 2.60383 18.298 2.07799 16.987 2.0666C15.676 2.0552 14.413 2.55918 13.47 3.46997L11.75 5.17997M14 11C13.5705 10.4259 13.0226 9.9508 12.3934 9.60704C11.7642 9.26328 11.0684 9.05886 10.3533 9.00765C9.63816 8.95643 8.92037 9.05961 8.24861 9.3102C7.57685 9.56079 6.96684 9.95291 6.45996 10.46L3.45996 13.46C2.54917 14.403 2.04519 15.666 2.05659 16.977C2.06798 18.288 2.59382 19.542 3.52086 20.4691C4.4479 21.3961 5.70197 21.922 7.01295 21.9334C8.32393 21.9447 9.58694 21.4408 10.53 20.53L12.24 18.82" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</a>
<span class="ts-share-notification" data-share-notification>Copied!</span>
</div>
</div>
</div>
</div>
</div>
</aside>
</div>
</div>
<!-- Next and Previous Post Pagination -->
<div style="margin-top: 5rem;">
<div class="ts-container">
<div class="ts-post-pagination">
<div>
<p class="ts-section-heading">Previous Post</p>
<div class="ts-post-unfurl">
<div class="ts-post-unfurl-content">
<div>
<a href="https://slack.engineering/autotransform-efficient-codebase-modification/">
<strong>AutoTransform: Efficient Codebase Modification</strong>
<span aria-label="(opens in new tab)"></span>
</a>
<p class="ts-post-unfurl-text">
Every codebase starts off small and modern. While it’s still small, the team can easily&hellip; </p>
</div>
<div class="ts-entry__post-meta">
<div class="ts-meta-date">July 21, 2022</div>
<div class="ts-meta-reading-time"><div class="ts-meta-reading-time">6 min read</div></div>
</div>
</div>
<div class="ts-post-unfurl-img">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2022/07/alex-knight-2EJCSULRwC8-unsplash.jpg?w=160&amp;h=160&amp;crop=1" class="attachment-thumbnail size-thumbnail wp-post-image" alt="A small, smiling robot" decoding="async" loading="lazy" srcset="https://slack.engineering/wp-content/uploads/sites/7/2022/07/alex-knight-2EJCSULRwC8-unsplash.jpg?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2022/07/alex-knight-2EJCSULRwC8-unsplash.jpg?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2022/07/alex-knight-2EJCSULRwC8-unsplash.jpg?resize=110,110 110w" sizes="auto, (max-width: 160px) 100vw, 160px" /> </div>
</div>
</div>
<div>
<p class="ts-section-heading">Next Post</p>
<div class="ts-post-unfurl">
<div class="ts-post-unfurl-content">
<a href="https://slack.engineering/buildrock-a-build-platform-at-slack/">
<strong>BuildRock: A Build Platform at Slack</strong>
<span aria-label="(opens in new tab)"></span>
</a>
<p class="ts-post-unfurl-text">
Our build platform is an essential piece of delivering code to production efficiently and safely&hellip; </p>
<div class="ts-entry__post-meta">
<div class="ts-meta-date">August 31, 2022</div>
<div class="ts-meta-reading-time"><div class="ts-meta-reading-time">12 min read</div></div>
</div>
</div>
<div class="ts-post-unfurl-img">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2022/07/business-concept-with-wood-blocks.jpg?w=160&amp;h=160&amp;crop=1" class="attachment-thumbnail size-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" srcset="https://slack.engineering/wp-content/uploads/sites/7/2022/07/business-concept-with-wood-blocks.jpg?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2022/07/business-concept-with-wood-blocks.jpg?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2022/07/business-concept-with-wood-blocks.jpg?resize=110,110 110w" sizes="auto, (max-width: 160px) 100vw, 160px" /> </div>
</div>
</div>
</div>
</div>
</div>
<div class="ts-entry__post-related">
<div class="ts-container">
<p class="ts-section-heading">Recommended Reading</p>
<div class="ts-entry__post-wrap">
<article class="post-17854 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-automation-testing tag-developer-productivity tag-innovation ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/06/agent_vs_traditional.png?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="Agentic vs. traditional testing paths" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/agentic-testing-where-agents-fit-in-the-e2e-testing-stack/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">June 11, 2026</div> <div class="ts-meta-reading-time">12 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/agentic-testing-where-agents-fit-in-the-e2e-testing-stack/"><span>Agentic Testing: Where Agents Fit in the E2E Testing Stack</span></a></h2><span class="ts-mention">@Sergii Gorbachov</span> </div>
</div>
</article>
<article class="post-17812 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-backend tag-cloud-computing tag-collaboration tag-engineering tag-infrastructure tag-innovation tag-machine-learning tag-software-development ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/05/pexels-photograph-6732010.jpg?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">May 28, 2026</div> <div class="ts-meta-reading-time">17 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/"><span>Slack AI: The Path to Multi-Cloud</span></a></h2><span class="ts-mention">@Shaurya Kethireddy</span> </div>
</div>
</article>
<article class="post-17732 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-golang tag-infrastructure tag-networking tag-observability tag-open-source ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/03/cover.png?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/from-custom-to-open-scalable-network-probing-and-http-3-readiness-with-prometheus/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">March 31, 2026</div> <div class="ts-meta-reading-time">5 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/from-custom-to-open-scalable-network-probing-and-http-3-readiness-with-prometheus/"><span>From Custom to Open: Scalable Network Probing and HTTP/3 Readiness with Prometheus</span></a></h2><span class="ts-mention">@Slack Engineering</span> </div>
</div>
</article>
<article class="post-17591 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-build-performance tag-caching tag-ci-cd tag-developer-experience tag-devops tag-python ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2025/10/DevXP-Council-Presentation6-e1759869480617.png?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="An example service build graph. It shows a set of Python files building into a backend artifact and a set of TypeScript files building into a frontend artifact." decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/build-better-software-to-build-software-better/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">November 6, 2025</div> <div class="ts-meta-reading-time">18 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/build-better-software-to-build-software-better/"><span>Build better software to build software better</span></a></h2><span class="ts-mention">@David Reed</span> </div>
</div>
</article>
</div>
</div>
</div>
<!-- Arc -->
<div class="c-arc concave-up">
<div class="c-arc__upper color-blue"></div>
<div class="c-arc__lower color-white"></div>
</div>
</div>
</main>
<footer class="ts-footer">
<div class="ts-container">
<div class="ts-footer__item">
<div class="ts-footer__col ts-col-left">
<div class="ts-footer__inner">
<div class="ts-logo">
<a class="ts-footer__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
</div>
<div class="ts-footer__col ts-col-canter">
<div class="ts-footer__inner">
<nav class="ts-footer__nav">
<div class="ts-footer__nav-item">
<ul class="ts-footer__nav-inner">
<li><a href="https://slack.engineering/articles/">Articles</a></li>
<!--<li><a href="https://slack.engineering/about/">About</a></li>-->
<li><a href="https://slack.com/careers/dept/software-engineering" target="_blank">Careers</a></li>
<li><a href="https://slack.dev/blog/" target="_blank">Slack Developer Blog</a></li>
<li><a href="https://slack.com/blog/" target="_blank">The Slack Blog</a></li>
</ul>
</div>
</nav>
</div>
</div>
<div class="ts-footer__col ts-col-right">
<div class="ts-footer__inner">
<span>
<a href="https://x.com/SlackEng" target="_blank" class="ts-share-buttons-twitter">
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M9.70487 6.85148L15.289 0.5H13.9657L9.11705 6.0149L5.24443 0.5H0.777832L6.63398 8.8395L0.777832 15.5H2.10116L7.22147 9.67608L11.3112 15.5H15.7778L9.70455 6.85148H9.70487ZM7.89239 8.91297L7.29905 8.08255L2.57797 1.47476H4.61052L8.42048 6.80746L9.01383 7.63788L13.9663 14.5696H11.9338L7.89239 8.91329V8.91297Z" fill="currentcolor"></path>
</svg>
</a>
<a href="https://www.youtube.com/@Slackhq" target="_blank" class="ts-share-buttons-youtube">
<svg width="21" height="14" viewBox="0 0 21 14" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M10.463 0C19.0721 0 20.1483 0 20.1483 7C20.1483 14 19.0721 14 10.463 14C1.85397 14 0.777832 14 0.777832 7C0.777832 0 1.85397 0 10.463 0ZM8.46143 2.7998C8.10631 2.7998 7.77271 3.0798 7.77271 3.4998V10.4998C7.77271 10.9521 8.11707 11.1998 8.46143 11.1998C8.60133 11.1998 8.7197 11.1567 8.82732 11.0921L14.3694 7.59211C14.5846 7.47365 14.6923 7.26904 14.6923 6.9998C14.6923 6.73057 14.5846 6.52596 14.3694 6.4075L8.82732 2.9075C8.70894 2.83211 8.59057 2.7998 8.46143 2.7998Z" fill="currentcolor"></path>
</svg>
</a>
</span>
</div>
</div>
</div>
<br/><br/>
<div class="ts-footer__item">
<p class="ts-footer__info">
<a href="https://slack.com/terms-of-service/user" target="_blank">Terms of Service</a>
<a href="https://slack.com/trust/privacy/privacy-policy" target="_blank">Privacy Information</a>
<a href="#" class="optanon-show-settings">Cookie Preferences</a>
<a href="https://www.salesforce.com/form/other/privacy-request/?_gl=1*1yvb56x*_gcl_au*OTY0OTg5MzcuMTcyNTkxMjE5Nw..*_ga*MTMzNjk3MzQ3MC4xNzE4MDQ0MzEy*_ga_QTJQME5M5D*MTcyNjI2OTEwNy40OS4xLjE3MjYyNjkyMzAuNS4wLjA." target="_blank">Your Privacy Choices</a>
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="30" height="14"><path d="M7.4 12.8h6.8l3.1-11.6H7.4C4.2 1.2 1.6 3.8 1.6 7s2.6 5.8 5.8 5.8z" style="fill-rule:evenodd;clip-rule:evenodd;fill:#fff"/><path d="M22.6 0H7.4c-3.9 0-7 3.1-7 7s3.1 7 7 7h15.2c3.9 0 7-3.1 7-7s-3.2-7-7-7zm-21 7c0-3.2 2.6-5.8 5.8-5.8h9.9l-3.1 11.6H7.4c-3.2 0-5.8-2.6-5.8-5.8z" style="fill-rule:evenodd;clip-rule:evenodd;fill:#06f"/><path d="M24.6 4c.2.2.2.6 0 .8L22.5 7l2.2 2.2c.2.2.2.6 0 .8-.2.2-.6.2-.8 0l-2.2-2.2-2.2 2.2c-.2.2-.6.2-.8 0-.2-.2-.2-.6 0-.8L20.8 7l-2.2-2.2c-.2-.2-.2-.6 0-.8.2-.2.6-.2.8 0l2.2 2.2L23.8 4c.2-.2.6-.2.8 0z" style="fill:#fff"/><path d="M12.7 4.1c.2.2.3.6.1.8L8.6 9.8c-.1.1-.2.2-.3.2-.2.1-.5.1-.7-.1L5.4 7.7c-.2-.2-.2-.6 0-.8.2-.2.6-.2.8 0L8 8.6l3.8-4.5c.2-.2.6-.2.9 0z" style="fill:#06f"/></svg>
<br/><br/>
&copy; 2026 Slack Technologies, LLC, a Salesforce company. All rights reserved. Various trademarks held by their respective owners.
</p>
</div>
</div>
</footer>
</div>
</div>
<a href="#top" data-scroll-to-top class="ts-scroll-to-top">
<svg class="ts-icon-up" width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M5 12H19M19 12L12 5M19 12L12 19" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
<span>scroll to top</span>
</a>
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/sites/7/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/tinyspeck/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script type="text/javascript" src="https://slack.engineering/_static/??-eJyNjksOwjAMRC9EaypVFBaIs4TESh0S14pdVdyeRsCGFcv5vNHAJp1f2JANJK+RWEEqaXFGHjLd3wrSx97LFfukB/gfzI5jJ7N8MWKf14DaMiououbFBQx9If6dthnL3jTipwr6BzhVNAWtvuFK1t7cynWYztPpOF6GMb0AWAxOQA==" ></script><script id="jetpack-stats-js-before">
_stq = window._stq || [];
_stq.push([ "view", {"v":"ext","blog":"195077423","post":"14817","tz":"-7","srv":"slack.engineering","hp":"vip","j":"1:16.1.3"} ]);
_stq.push([ "clickTrackerInit", "195077423", "14817" ]);
//# sourceURL=jetpack-stats-js-before
</script>
<script data-wp-strategy="defer" defer fetchpriority="low" id="jetpack-stats-js" src="https://stats.wp.com/e-202637.js"></script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://slack.engineering/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://slack.engineering/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>