Files
nexus/sreweekly/articles/499/07-advancing-our-chef-infrastructure-safety-without-disruption.html
2026-09-12 17:23:01 +08:00

739 lines
77 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<html lang="en-US">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, height=device-height, initial-scale=1.0, maximum-scale=1">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<link rel="profile" href="https://gmpg.org/xfn/11" />
<link rel="shortcut icon" href="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/icons/favicon.png" sizes="16x16 32x32 48x48" type="image/png">
<!-- Optanon Consent Notice start -->
<script src="https://d34u8crftukxnk.cloudfront.net/onetrust/slack.engineering/production/scripttemplates/otSDKStub.js" type="text/javascript" charset="UTF-8" data-domain-script="019872bf-9ff8-7d80-8158-f0c910bcdfad"></script>
<!-- Optanon Consent Notice end -->
<title>Advancing Our Chef Infrastructure: Safety Without Disruption | Engineering at Slack</title>
<meta name="description" content="This post builds on our earlier work modernising Slack’s Chef infrastructure. Instead of a disruptive migration to Policyfiles, we focused on practical improvements to our existing EC2 and Chef frameworks - delivering safer, more reliable deploys with minimal change for our service owners.">
<meta property="og:url" content="https://slack.engineering/advancing-our-chef-infrastructure-safety-without-disruption/">
<meta property="og:type" content="website">
<meta property="og:title" content="Advancing Our Chef Infrastructure: Safety Without Disruption">
<meta property="og:description" content="This post builds on our earlier work modernising Slack’s Chef infrastructure. Instead of a disruptive migration to Policyfiles, we focused on practical improvements to our existing EC2 and Chef frameworks - delivering safer, more reliable deploys with minimal change for our service owners.">
<meta property="og:image" content="https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg">
<meta property="og:image:width" content="6000">
<meta property="og:image:height" content="4000">
<meta name="twitter:card" content="summary_large_image">
<meta property="twitter:domain" content="slack.engineering">
<meta property="twitter:url" content="https://slack.engineering/advancing-our-chef-infrastructure-safety-without-disruption/">
<meta name="twitter:title" content="Advancing Our Chef Infrastructure: Safety Without Disruption">
<meta name="twitter:description" content="This post builds on our earlier work modernising Slack’s Chef infrastructure. Instead of a disruptive migration to Policyfiles, we focused on practical improvements to our existing EC2 and Chef frameworks - delivering safer, more reliable deploys with minimal change for our service owners.">
<meta name="twitter:image" content="https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg">
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-590Q1VHB01"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-590Q1VHB01');
</script>
<!-- End Google Analytics -->
<title>Advancing Our Chef Infrastructure: Safety Without Disruption &#8211; Engineering at Slack</title>
<meta name='robots' content='max-image-preview:large' />
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://slack.engineering/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure-safety-without-disruption%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://slack.engineering/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure-safety-without-disruption%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='all-css-12' href='https://slack.engineering/_static/??-eJxtjEkOgkAQRS8klopGXRjP0hQlVHoMvwjh9ja61OUf3qOlNJyTSTIqYR40gRAc+6196UCwNQhhzJNx7gV7Bnb0hyqTIjpTpqDdN1H9ko0Spcneaa/uh/6s9aRpRRH25ACxapu7UFWbAGqycc/4OF5vbXs53Q/nN25bRHg=' type='text/css' media='all' />
<script type="text/javascript" src="https://slack.engineering/_static/??-eJyFi0sOgCAQQy+kjiRGdGE8i0HEQRiUT4y3FxL3rprXvsJ91sJRlBThNEkhBQhmEUdpN1SgM+/OR+FWGRodKsgPJGFS5rLqK0n/fNFYpF+ptqj8EuUnz3ZifOB9242s0y+jLDUm" ></script><link rel="https://api.w.org/" href="https://slack.engineering/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://slack.engineering/wp-json/wp/v2/posts/17611" /> <style>img#wpstats{display:none}</style>
<link rel="icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=32" sizes="32x32" />
<link rel="icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=192" sizes="192x192" />
<link rel="apple-touch-icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=180" />
<meta name="msapplication-TileImage" content="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=270" />
</head>
<body class="wp-singular post-template-default single single-post postid-17611 single-format-standard wp-embed-responsive wp-theme-tinyspeck">
<a class="ts-skip-link" href="#main">Skip to main content</a>
<div class="ts-site">
<div class="ts-site-inner">
<div class="ts-header-before"></div>
<header class="ts-header">
<div class="ts-container">
<div class="ts-header__inner ts-header__inner-desktop">
<div class="ts-header__col ts-col-left">
<div class="ts-logo">
<a class="ts-header__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
<div class="ts-header__col ts-col-center"></div>
<div class="ts-header__col ts-col-right">
<!--<a href="https://slack.engineering/articles/">Articles</a>
<a href="https://slack.engineering/about/">About</a>-->
<span class="ts-header__search-toggle" role="button" data-search-toggle="open">
<span class="ts-header__search-toggle-label">Search</span>
<svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</span>
</div>
</div>
<div class="ts-header__inner ts-header__inner-mobile">
<div class="ts-header__col ts-col-left"></div>
<div class="ts-header__col ts-col-center">
<div class="ts-logo">
<a class="ts-header__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
<div class="ts-header__col ts-col-right">
<span class="ts-header__search-toggle" role="button" data-search-toggle="open">
<span class="ts-header__search-toggle-label">Search</span>
<svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</span>
</div>
</div>
</div>
<div class="ts-search">
<div class="ts-container">
<div class="ts-search__close">
<div class="ts-search__close-inner" role="button" data-search-toggle="close">
<span class="ts-search__close-label">Close</span>
<svg class="ts-search__close-icon ts-icon ts-icon-x" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M18 6L6 18M6 6L18 18" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</div>
</div>
<div class="ts-search__wrapper">
<form role="search" method="get" class="ts-search__nav-form" action="https://slack.engineering/">
<div class="ts-search__group">
<input class="ts-search__input" type="search" required value="" name="s" placeholder="What are you looking for?">
<button class="ts-search__submit">Search</button>
</div>
</form>
</div>
</div>
</div>
</header>
<main id="main" class="ts-site-primary">
<div class="ts-site-content ts-sidebar-enabled ts-singular-thumbnail-enabled">
<div class="ts-container">
<div class="ts-main-content">
<div class="ts-content-area">
<div class="ts-entry__header ts-entry__header-large">
<div class="ts-entry__header-inner">
<div class="ts-entry__header-info">
<div class="ts-entry__post-meta-wrapper">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">October 23, 2025</div>
<div class="ts-meta-reading-time">15 min read</div> </div>
</div>
<h1 class="ts-jumbo"><span>Advancing Our Chef Infrastructure: Safety Without Disruption</span></h1> <p class="ts-hero-paragraph">Building a safer, more reliable path forward for Chef at Slack</p>
<div class="c-article__author__meta"><div class="c-article__author"><div class="c-article__author__avatar"><img src="https://slack.engineering/wp-content/uploads/sites/7/2020/09/Screenshot-2023-03-01-at-9.02.01-am.png"/></div><div class="c-article__author__info"><span class="c-article__author__name">Archie Gunasekara</span><span class="c-article__author__title">Staff Software Engineer, Cloud</span></div></div></div> </div>
<figure class="ts-entry__post-media post-media filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg?w=1020" class="attachment-post-thumbnail size-post-thumbnail wp-post-image" alt="person slicing green vegetable on chopping board" decoding="async" fetchpriority="high" srcset="https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg 1600w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg?resize=640,360 640w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg?resize=768,432 768w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg?resize=1280,720 1280w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg?resize=1536,864 1536w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg?resize=380,214 380w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg?resize=800,450 800w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/8_tlng54n8y.jpg?resize=1160,653 1160w" sizes="(max-width: 1020px) 100vw, 1020px" /> </figure>
</div>
</div>
<div class="ts-entry__wrap">
<div class="ts-entry__container">
<aside class="ts-widget-area ts-sidebar__area">
<div class="ts-sidebar__inner">
<!-- Search -->
<div class="widget widget_search">
<p class="ts-section-heading">Search</p>
<form role="search" method="get" class="ts-search__form" action="https://slack.engineering/">
<div class="ts-search__container">
<input required class="ts-search__input" type="search" value="" name="s" placeholder="">
<button class="ts-search__submit"><svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg></button>
</div>
</form>
</div>
<!-- Latest Posts -->
<div class="widget">
<p class="ts-section-heading">Latest Posts</p>
<div class="widget-body ts-widget-posts">
<ul>
<li class="ts-post-item">
<article class="post-17832 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-migration ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">July 14, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/">Shipyard: How We Built Slack’s Next-Generation EC2 Platform</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17854 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-automation-testing tag-developer-productivity tag-innovation ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">June 11, 2026</div> <div class="ts-meta-reading-time">12 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/agentic-testing-where-agents-fit-in-the-e2e-testing-stack/">Agentic Testing: Where Agents Fit in the E2E Testing Stack</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17812 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-backend tag-cloud-computing tag-collaboration tag-engineering tag-infrastructure tag-innovation tag-machine-learning tag-software-development ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">May 28, 2026</div> <div class="ts-meta-reading-time">17 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/">Slack AI: The Path to Multi-Cloud</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17774 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-airflow tag-aws tag-big-data tag-data-engineering tag-data-infrastructure tag-security ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">May 5, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/">From SSH to REST: A Security-Driven Modernization of Slack&#8217;s EMR Data Pipelines</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17752 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-development tag-security tag-software-engineering ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">April 13, 2026</div> <div class="ts-meta-reading-time">14 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/managing-context-in-long-run-agentic-applications/">Managing context in long-run agentic applications</a>
</h2>
</div>
</div>
</article>
</li>
</ul>
</div>
</div>
<!-- Archives -->
<div class="widget">
<p class="ts-section-heading">Archives</p>
<ul class="wp-block-archives-list wp-block-archives">
<li><a href='https://slack.engineering/2026/'>2026</a>&nbsp;(7)</li>
<li><a href='https://slack.engineering/2025/'>2025</a>&nbsp;(9)</li>
<li><a href='https://slack.engineering/2024/'>2024</a>&nbsp;(19)</li>
<li><a href='https://slack.engineering/2023/'>2023</a>&nbsp;(16)</li>
<li><a href='https://slack.engineering/2022/'>2022</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2021/'>2021</a>&nbsp;(24)</li>
<li><a href='https://slack.engineering/2020/'>2020</a>&nbsp;(26)</li>
<li><a href='https://slack.engineering/2019/'>2019</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2018/'>2018</a>&nbsp;(11)</li>
<li><a href='https://slack.engineering/2017/'>2017</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2016/'>2016</a>&nbsp;(19)</li>
</ul>
</div>
</div>
</aside>
<div class="ts-entry__content-wrap">
<div class="ts-single-post ts-wysiwyg">
<p>Last year, I wrote a blog post titled <a href="https://slack.engineering/advancing-our-chef-infrastructure/">Advancing Our Chef Infrastructure</a>, where we explored the evolution of our Chef infrastructure over the years. We talked about the shift from a single Chef stack to a multi-stack model, and the challenges that came with it &#8211; from updating how we handle cookbook uploads to navigating the limitations around Chef searches.<br />
If you haven’t had a chance to read that post yet, I highly recommend checking it out first to get the full context for this post.</p>
<p>At Slack, keeping our service reliable is always the top priority. In my <a href="https://slack.engineering/advancing-our-chef-infrastructure/">last post</a>, I talked about the first phase of our work to make Chef and EC2 provisioning safer. With that behind us, we started looking at what else we could do to make deploys even safer and more reliable.</p>
<p>One idea we explored was moving to Chef Policyfiles. That would have meant replacing roles and environments and asking dozens of teams to change their cookbooks. In the long run, it might have made things safer, but in the short term it would have been a huge effort and added more risk than it solved.<br />
So instead, this post is about the path we chose: improving our existing EC2 framework in a way that doesn’t disrupt cookbooks or roles, while still giving us more safety in our Chef deployments.</p>
<h2>Splitting Chef Environments</h2>
<p>Previously, each instance had a cron job that triggered a Chef run every few hours on a set schedule. These scheduled runs were primarily for compliance purposes — to ensure our fleet remained in a consistent and defined configuration state. To reduce risk, the timing of these cron jobs was staggered across availability zones, helping us avoid running Chef on all nodes simultaneously. This strategy gave us a buffer: if a bad change was introduced, it would only impact a subset of nodes initially, giving us a chance to catch and fix the issue before it spread.<br />
However, this approach had a critical limitation. With a single shared production environment, even if Chef wasn’t running everywhere at once, any newly provisioned nodes would immediately pick up the latest (possibly bad) changes from that shared environment. This became a significant reliability risk, especially during large scale-out events, where dozens or hundreds of nodes could start up with a broken configuration.<br />
To address this, we split our single production Chef environment into multiple buckets: <code>prod-1</code>, <code>prod-2</code>, &#8230;, <code>prod-6</code>. Service teams could still launch instances as “prod,” but behind the scenes we mapped each instance to one of these more specific environments based on its Availability Zone. That way, new nodes no longer pulled from one global source of truth &#8211; they were evenly distributed across isolated environments, which added an extra layer of safety and resilience.</p>
<p>At Slack, our base AMIs include a tool called <code>Poptart Bootstrap</code>, which is baked in during the AMI build process. This tool runs via cloud-init at instance boot time and is responsible for tasks such as creating the node&#8217;s Chef object, setting up any required DNS entries, and posting a success or failure message to Slack (with customisable channels based on the team that owns the node).</p>
<p>To support the environment split, we extended <code>Poptart Bootstrap</code> to include logic that inspects the node’s AZ ID and assigns it to one of the numbered production Chef environments. This change allowed us to stop pointing all production nodes at a single shared Chef environment and instead spread them across multiple environments.</p>
<p>Previously, changing the cookbook versions in the single prod environment affected the entire fleet. With this new bucketed approach, we gained the flexibility to update each environment independently. Now, a change to a single environment only impacts nodes in specific AZs, significantly reducing risk and blast radius during deployments.</p>
<p><img decoding="async" width="438" height="597" class="alignnone size-medium wp-image-17612" src="https://slack.engineering/wp-content/uploads/sites/7/2025/10/env_flow.png?w=438" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2025/10/env_flow.png 438w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/env_flow.png?resize=380,518 380w" sizes="(max-width: 438px) 100vw, 438px" /></p>
<p>With this approach, we promote the latest cookbook changes to the sandbox environments at the top of the hour. These changes are then managed by a Kubernetes cron job and rolled out to the dev environments at the top of the hour, and finally begin rolling out to production environments starting at 30 minutes past the hour.</p>
<p><code>prod-1</code> is treated as our canary production environment. It receives updates every hour (assuming there have been new changes merged into the cookbooks) to ensure that changes are regularly exercised in a real production setting &#8211; alongside our sandbox and dev environments. This enables us to catch production-impacting issues early and in smaller, safer increments.</p>
<p>Changes to <code>prod-2</code> through <code>prod-6</code> are rolled out using a release train model. Before a new version is deployed to <code>prod-2</code>, we ensure the previous version has successfully progressed through all production environments up to <code>prod-6</code>. This staggered rollout minimises blast radius and gives us the opportunity to catch regressions earlier in the release train.<br />
If we waited for a version to pass through all environments before updating <code>prod-1</code> &#8211; as we do with <code>prod-2</code> onward &#8211; we’d end up testing artifacts with large, cumulative changes. By contrast, updating prod-1 frequently with the latest version allows us to detect issues closer to when they were introduced.</p>
<p>Here’s an example showing how artifact version A moves through each environment in this release cycle:</p>
<h3>Hour X:00</h3>
<pre><code class="language-vim">Sandbox ---&gt; Version A (Latest)
Dev ---&gt; Version A (Latest)
Prod-1 (no change - currently at version Z)
Prod-2 (no change - currently at version Z)
Prod-3 (no change - currently at version Z)
Prod-4 (no change - currently at version Z)
Prod-5 (no change - currently at version Z)
Prod-6 (no change - currently at version Z)</code></pre>
<h3>Hour X:30</h3>
<p>(Now that Prod-2 to Prod-6 are all on the same version, we’ll begin updating them again)</p>
<pre><code class="language-vim">Sandbox (no change - currently at version A)
Dev (no change - currently at version A)
Prod-1 ---&gt; Version A (dev version)
Prod-2 ---&gt; Version A (prod-1 version)
Prod-3 (no change - currently at version Z)
Prod-4 (no change - currently at version Z)
Prod-5 (no change - currently at version Z)
Prod-6 (no change - currently at version Z)</code></pre>
<p>Someone committed a change, resulting in the creation of a new artifact called B.</p>
<h3>Hour (X + 1):00</h3>
<pre><code class="language-vim">Sandbox ---&gt; Version B (Latest)
Dev ---&gt; Version B (Latest)
Prod-1 (no change - currently at version A)
Prod-2 (no change - currently at version A)
Prod-3 (no change - currently at version Z)
Prod-4 (no change - currently at version Z)
Prod-5 (no change - currently at version Z)
Prod-6 (no change - currently at version Z)</code></pre>
<h3>Hour (X + 1):30</h3>
<pre><code class="language-vim">Sandbox (no change - currently at version B)
Dev (no change - currently at version B)
Prod-1 ---&gt; Version B (dev version)
Prod-2 (no change - currently at version A)
Prod-3 ---&gt; Version A (prod-2 version)
Prod-4 (no change - currently at version Z)
Prod-5 (no change - currently at version Z)
Prod-6 (no change - currently at version Z)</code></pre>
<p>Someone committed a change, resulting in the creation of a new artifact called C.</p>
<h3>Hour (X + 2):00</h3>
<pre><code class="language-vim">Sandbox ---&gt; Version C (Latest)
Dev ---&gt; Version C (Latest)
Prod-1 (no change - currently at version B)
Prod-2 (no change - currently at version A)
Prod-3 (no change - currently at version A)
Prod-4 (no change - currently at version Z)
Prod-5 (no change - currently at version Z)
Prod-6 (no change - currently at version Z)</code></pre>
<h3>Hour (X + 2):30</h3>
<pre><code class="language-vim">Sandbox (no change - currently at version C)
Dev (no change - currently at version C)
Prod-1 ---&gt; Version C (dev version)
Prod-2 (no change - currently at version A)
Prod-3 (no change - currently at version A)
Prod-4 ---&gt; Version A (prod-3 version)
Prod-5 (no change - currently at version Z)
Prod-6 (no change - currently at version Z)</code></pre>
<p>Someone committed a change, resulting in the creation of a new artifact called D.</p>
<h3>Hour (X + 3):00</h3>
<pre><code class="language-vim">Sandbox ---&gt; Version D (Latest)
Dev ---&gt; Version D (Latest)
Prod-1 (no change - currently at version C)
Prod-2 (no change - currently at version A)
Prod-3 (no change - currently at version A)
Prod-4 (no change - currently at version A)
Prod-5 (no change - currently at version Z)
Prod-6 (no change - currently at version Z)</code></pre>
<h3>Hour (X + 3):30</h3>
<pre><code class="language-vim">Sandbox (no change - currently at version D)
Dev (no change - currently at version D)
Prod-1 ---&gt; Version D (dev version)
Prod-2 (no change - currently at version A)
Prod-3 (no change - currently at version A)
Prod-4 (no change - currently at version A)
Prod-5 ---&gt; Version A (prod-4 version)
Prod-6 (no change - currently at version Z)</code></pre>
<p>Someone committed a change, resulting in the creation of a new artifact called E.</p>
<h3>Hour (X + 4):00</h3>
<pre><code class="language-vim">Sandbox ---&gt; Version E (Latest)
Dev ---&gt; Version E (Latest)
Prod-1 (no change - currently at version D)
Prod-2 (no change - currently at version A)
Prod-3 (no change - currently at version A)
Prod-4 (no change - currently at version A)
Prod-5 (no change - currently at version A)
Prod-6 (no change - currently at version Z)</code></pre>
<h3>Hour (X + 4):30</h3>
<pre><code class="language-vim">Sandbox (no change - currently at version E)
Dev (no change - currently at version E)
Prod-1 ---&gt; Version E (dev version)
Prod-2 (no change - currently at version A)
Prod-3 (no change - currently at version A)
Prod-4 (no change - currently at version A)
Prod-5 (no change - currently at version A)
Prod-6 ---&gt; Version A (prod-5 version)</code></pre>
<p>Someone committed a change, resulting in the creation of a new artifact called F.</p>
<h3>Hour (X + 5):00</h3>
<pre><code class="language-vim">Sandbox ---&gt; Version F (Latest)
Dev ---&gt; Version F (Latest)
Prod-1 (no change - currently at version E)
Prod-2 (no change - currently at version A)
Prod-3 (no change - currently at version A)
Prod-4 (no change - currently at version A)
Prod-5 (no change - currently at version A)
Prod-6 (no change - currently at version A)</code></pre>
<p>Now that Prod-2 to Prod-6 are all on the same version, we’ll begin updating them again and this cycle will continue till the end of time.</p>
<h3>Hour (X + 5):30</h3>
<pre><code class="language-vim">Sandbox (no change - currently at version F)
Dev (no change - currently at version F)
Prod-1 ---&gt; Version F (dev version)
Prod-2 ---&gt; Version F (prod-1 version)
Prod-3 (no change - currently at version A)
Prod-4 (no change - currently at version A)
Prod-5 (no change - currently at version A)
Prod-6 (no change - currently at version A)</code></pre>
<p>As you can see above, changes now take a bit longer to roll out across all our production nodes. However, this delay provides valuable time between deployments across different availability zones, allowing us to catch and address any issues before problematic changes are fully propagated.</p>
<p>By introducing split production environments (e.g., <code>prod-1</code> to <code>prod-6</code>), we&#8217;ve been able to sidestep the old problem of new nodes pulling in a bad config the moment they came online. Now, each node is tied to the environment for its AZ, and changes roll out gradually instead of everywhere at once. That means if something goes wrong in one AZ, we can keep scaling safely in the others while we fix the issue.</p>
<p>It’s a much more resilient setup than before &#8211; we’ve taken away the single point of failure and built in guardrails that make the whole system safer and more predictable.</p>
<h2>What changed in the way we trigger Chef?</h2>
<p>Now that we have multiple production Chef environments, each one receives updates at different times. For example, if a version is currently in the middle of being rolled out, the next version must wait until that rollout is fully completed across all production environments. On the other hand, when there are no updates in progress, a new version can be rolled out to all environments more quickly.</p>
<p>Because of this variability, triggering Chef via a fixed cron schedule is no longer practical &#8211; we can’t reliably predict when a given Chef environment will receive new changes. As a result, we’ve moved away from scheduled runs and instead built a new service that triggers Chef runs on nodes based on signals. This service ensures Chef only runs when there are actual updates available, improving both safety and efficiency.</p>
<h3>Chef Summoner</h3>
<p><img decoding="async" width="768" height="768" class="alignnone size-medium wp-image-17614" src="https://slack.engineering/wp-content/uploads/sites/7/2025/10/summoner_logo.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2025/10/summoner_logo.png 768w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/summoner_logo.png?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/summoner_logo.png?resize=640,640 640w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/summoner_logo.png?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/summoner_logo.png?resize=110,110 110w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/summoner_logo.png?resize=380,380 380w" sizes="(max-width: 768px) 100vw, 768px" /></p>
<p>If you remember from my previous blog post, we built a service called Chef Librarian that watches for new Chef cookbook artifacts and uploads them to all of our Chef stacks. It also exposes an API endpoint that allows us to promote a specific version of an artifact to a given environment.</p>
<p>We recently enhanced Chef Librarian to send a message to an S3 bucket whenever it promotes a version of an artifact to an environment. The contents of that S3 bucket look like this:</p>
<pre><code class="language-vim">agunasekara@z-ops-agunasekara-iad-dinosaur:~ &gt;&gt; s3-tree BUCKET_NAME
BUCKET_NAME
└── chef-run-triggers
├── basalt
│ ├── ami-dev
│ ├── ami-prod
│ ├── dev
│ ├── prod
│ ├── prod-1
│ ├── prod-2
│ ├── prod-3
│ ├── prod-4
│ ├── prod-5
│ ├── prod-6
│ └── sandbox
└── ironstone
├── ami-dev
├── ami-prod
├── dev
├── prod
├── prod-1
├── prod-2
├── prod-3
├── prod-4
├── prod-5
├── prod-6
└── sandbox</code></pre>
<p>Under the chef-run-triggers key, we maintain a nested structure where each key represents one of our Chef stacks. Within each stack key, there are additional keys for each environment name.</p>
<p>Each of these environment keys contains a JSON object with contents similar to the example below:</p>
<pre><code class="language-json">{
&quot;Splay&quot;: 15,
&quot;Timestamp&quot;: &quot;2025-07-28T02:02:31.054989714Z&quot;,
&quot;ManifestRecord&quot;: {
&quot;version&quot;: &quot;20250728.1753666491.0&quot;,
&quot;chef_shard&quot;: &quot;basalt&quot;,
&quot;datetime&quot;: 1753666611,
&quot;latest_commit_hash&quot;: &quot;XXXXXXXXXXXXXX&quot;,
&quot;manifest_content&quot;: {
&quot;base_version&quot;: &quot;20250728.1753666491.0&quot;,
&quot;latest_commit_hash&quot;: &quot;XXXXXXXXXXXXXX&quot;,
&quot;author&quot;: &quot;Archie Gunasekara &lt;agunasekara@slack-corp.com&gt;&quot;,
&quot;cookbook_versions&quot;: {
&quot;apt&quot;: &quot;7.5.23&quot;,
...
&quot;aws&quot;: &quot;9.2.1&quot;
},
&quot;site_cookbook_versions&quot;: {
&quot;apache2&quot;: &quot;20250728.1753666491.0&quot;,
...
&quot;squid&quot;: &quot;20250728.1753666491.0&quot;
}
},
&quot;s3_bucket&quot;: &quot;BUCKET_NAME&quot;,
&quot;s3_key&quot;: &quot;20250728.1753666491.0.tar.gz&quot;,
&quot;ttl&quot;: 1756085811,
&quot;upload_complete&quot;: true
}
}</code></pre>
<p>Next, we built a service called Chef Summoner. This service runs on every node at Slack and is responsible for checking the S3 key corresponding to the node’s Chef stack and environment. If a new version is present in the signal sent out by Chef Librarian, the service reads the configured splay value and schedules a Chef run accordingly.</p>
<p>The splay is used to stagger Chef runs so that not all nodes in a given environment and stack try to run Chef at the same time. This helps avoid spikes in load and resource contention. We can also customize the splay depending on our needs &#8211; for example, when we trigger a Chef run using a custom signal from Librarian and want to spread the runs out more intentionally.</p>
<p>However, if no changes are merged and no new Chef artifacts are built, Librarian has nothing to promote, and no new signals are sent out. Despite that, we still need to ensure Chef runs at least once every 12 hours to maintain compliance and ensure nodes stay in their expected configuration state. So, Chef Summoner will also run Chef at least once every 12 hours, even if no new signals have been received.</p>
<p>The Summoner service keeps track of its own state locally &#8211; this includes things like the last run time and the artifact version used &#8211; so it can compare any new signals with the most recent run and determine whether a new Chef run is required.</p>
<p>The overall flow looks like this:</p>
<p><img loading="lazy" decoding="async" width="630" height="1052" class="alignnone size-medium wp-image-17616" src="https://slack.engineering/wp-content/uploads/sites/7/2025/10/summoner_flow.png?w=630" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2025/10/summoner_flow.png 630w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/summoner_flow.png?resize=380,635 380w" sizes="auto, (max-width: 630px) 100vw, 630px" /></p>
<p>Now that Chef Summoner is the primary mechanism we rely on to trigger Chef runs, it becomes a critical piece of infrastructure. After a node is provisioned, subsequent Chef runs are responsible for keeping Chef Summoner itself up to date with the latest changes.</p>
<p>But if we accidentally roll out a broken version of Chef Summoner, it may stop triggering Chef runs altogether &#8211; making it impossible to roll out a fixed version using our normal deployment flow.</p>
<p>To mitigate this, we bake in a fallback cron job on every node. This cron job checks the local state that Chef Summoner stores (e.g., last run time and artifact version) and ensures Chef has been run at least once every 12 hours. If the cron job detects that Chef Summoner has failed to run Chef in that timeframe, it will trigger a Chef run directly. This gives us a recovery path to push a working version of Chef Summoner back out.</p>
<p>In addition to this safety net, we also have tooling that allows us to trigger ad hoc Chef runs across the fleet or a subset of nodes when needed.</p>
<h2>What’s Next?</h2>
<p>All of these recent changes to our EC2 ecosystem have made rolling out infrastructure changes significantly safer. Teams no longer have to worry about a bad update impacting the entire fleet. While we’ve come a long way, the platform is still not perfect.</p>
<p>One major limitation is that we still can’t easily support service-level deployments. In theory, we could create a dedicated set of Chef environments for each service and promote artifacts individually &#8211; but with the hundreds of services we operate at Slack, this quickly becomes unmanageable at scale.</p>
<p>With that in mind, we’ve decided to mark our legacy EC2 platform as feature complete and move it into maintenance mode. In its place, we’re building a brand-new EC2 ecosystem called Shipyard, designed specifically for teams that can’t yet move to our container-based platform, Bedrock.</p>
<p>Shipyard isn’t just an iteration of our old system &#8211; it’s a complete reimagining of how EC2-based services should work. It introduces concepts like service-level deployments, metric-driven rollouts, and fully automated rollbacks when things go wrong.</p>
<p>We’re currently building a Shipyard and targeting a soft launch this quarter, with plans to onboard our first two teams for testing and feedback. I’m excited to share more about Shipyard in my next blog post &#8211; stay tuned!</p>
</div>
<div class="ts-tags-section"><span class="ts-tag">#<a href="https://slack.engineering/tags/aws/" rel="tag">aws</a></span></div> </div>
</div>
</div>
</div>
<aside class="ts-metabar__area">
<div class="ts-metabar__inner">
<div class="ts-metabar__item">
<div class="ts-share-buttons-wrap">
<div class="ts-share-buttons-items">
<div class="ts-share-buttons-item ts-share-buttons-twitter">
<a href="https://x.com/intent/post?url=https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure-safety-without-disruption%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-twitter" width="21" viewBox="0 0 72 65" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M22.6891 0H0L26.9014 35.3477L1.72089 65H13.355L32.3999 42.5731L49.3109 64.794H72L44.317 28.4191L44.366 28.4818L68.2015 0.412927H56.5674L38.8667 21.2573L22.6891 0ZM12.524 6.19052H19.5874L59.476 58.603H52.4126L12.524 6.19052Z" fill="currentColor"/></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-linkedin">
<a href="https://www.linkedin.com/shareArticle?mini=true&#038;url=https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure-safety-without-disruption%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-linkedin" width="21" height="21" viewBox="0 0 70 70" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M15.8066 23.0188H1.17969V69.663H15.8066V23.0188Z" fill="currentColor"/><path d="M55.3732 22.0077C54.834 21.9403 54.2611 21.9066 53.6881 21.8729C45.4984 21.5359 40.8812 26.389 39.2635 28.4786C38.8254 29.0515 38.6231 29.3886 38.6231 29.3886V23.1536H24.6366V69.7978H38.6231H39.2635C39.2635 65.0458 39.2635 60.3274 39.2635 55.5754C39.2635 53.014 39.2635 50.4526 39.2635 47.8912C39.2635 44.7232 39.0276 41.3529 40.6116 38.4545C41.9597 36.0279 44.3862 34.8147 47.1161 34.8147C55.2047 34.8147 55.3732 42.1281 55.3732 42.8021C55.3732 42.8358 55.3732 42.8695 55.3732 42.8695V70H70.0001V39.5667C70.0001 29.1526 64.7088 23.0188 55.3732 22.0077Z" fill="currentColor"/><path d="M8.49301 16.986C13.1836 16.986 16.9861 13.1836 16.9861 8.49303C16.9861 3.80246 13.1836 0 8.49301 0C3.80244 0 0 3.80246 0 8.49303C0 13.1836 3.80244 16.986 8.49301 16.986Z" fill="currentColor"/></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-facebook">
<a href="https://www.facebook.com/sharer/sharer.php?u=https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure-safety-without-disruption%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-facebook" width="21" height="21" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><g clip-path="url(#clip0_11_112)"><path d="M12 0C18.6274 0 24 5.37259 24 12C24 18.1352 19.3955 23.1944 13.4538 23.9121V15.667L16.7001 15.667L17.3734 12H13.4538V10.7031C13.4538 9.73417 13.6439 9.06339 14.0799 8.63483C14.5159 8.20627 15.1979 8.01993 16.1817 8.01993C16.4307 8.01993 16.6599 8.02241 16.8633 8.02736C17.1591 8.03456 17.4002 8.047 17.568 8.06467V4.74048C17.501 4.72184 17.4218 4.70321 17.3331 4.68486C17.1321 4.6433 16.8822 4.60324 16.6136 4.56806C16.0523 4.49453 15.4093 4.4423 14.9594 4.4423C13.1424 4.4423 11.7692 4.83102 10.8107 5.63619C9.65388 6.60791 9.10108 8.18622 9.10108 10.4199V12H6.62659V15.667H9.10108V23.6466C3.87432 22.3498 0 17.6277 0 12C0 5.37259 5.37259 0 12 0Z" fill="currentColor"/></g><defs><clipPath id="clip0_11_112"><rect width="24" height="24" fill="white"/></clipPath></defs></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-threads">
<a href="https://www.threads.net/intent/post?text=+https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure-safety-without-disruption%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon" width="21" viewBox="0 0 448 512" xmlns="http://www.w3.org/2000/svg"><path d="M331.5 235.7c2.2 .9 4.2 1.9 6.3 2.8c29.2 14.1 50.6 35.2 61.8 61.4c15.7 36.5 17.2 95.8-30.3 143.2c-36.2 36.2-80.3 52.5-142.6 53h-.3c-70.2-.5-124.1-24.1-160.4-70.2c-32.3-41-48.9-98.1-49.5-169.6V256v-.2C17 184.3 33.6 127.2 65.9 86.2C102.2 40.1 156.2 16.5 226.4 16h.3c70.3 .5 124.9 24 162.3 69.9c18.4 22.7 32 50 40.6 81.7l-40.4 10.8c-7.1-25.8-17.8-47.8-32.2-65.4c-29.2-35.8-73-54.2-130.5-54.6c-57 .5-100.1 18.8-128.2 54.4C72.1 146.1 58.5 194.3 58 256c.5 61.7 14.1 109.9 40.3 143.3c28 35.6 71.2 53.9 128.2 54.4c51.4-.4 85.4-12.6 113.7-40.9c32.3-32.2 31.7-71.8 21.4-95.9c-6.1-14.2-17.1-26-31.9-34.9c-3.7 26.9-11.8 48.3-24.7 64.8c-17.1 21.8-41.4 33.6-72.7 35.3c-23.6 1.3-46.3-4.4-63.9-16c-20.8-13.8-33-34.8-34.3-59.3c-2.5-48.3 35.7-83 95.2-86.4c21.1-1.2 40.9-.3 59.2 2.8c-2.4-14.8-7.3-26.6-14.6-35.2c-10-11.7-25.6-17.7-46.2-17.8H227c-16.6 0-39 4.6-53.3 26.3l-34.4-23.6c19.2-29.1 50.3-45.1 87.8-45.1h.8c62.6 .4 99.9 39.5 103.7 107.7l-.2 .2zm-156 68.8c1.3 25.1 28.4 36.8 54.6 35.3c25.6-1.4 54.6-11.4 59.5-73.2c-13.2-2.9-27.8-4.4-43.4-4.4c-4.8 0-9.6 .1-14.4 .4c-42.9 2.4-57.2 23.2-56.2 41.8l-.1 .1z"></path></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-copy">
<a href="#" class="ts-share-buttons-link" data-share-link title="Copy link">
<svg width="21" height="21" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M10 13C10.4295 13.5741 10.9774 14.0491 11.6066 14.3929C12.2357 14.7367 12.9315 14.9411 13.6467 14.9923C14.3618 15.0435 15.0796 14.9403 15.7513 14.6897C16.4231 14.4392 17.0331 14.047 17.54 13.54L20.54 10.54C21.4508 9.59695 21.9548 8.33394 21.9434 7.02296C21.932 5.71198 21.4061 4.45791 20.4791 3.53087C19.5521 2.60383 18.298 2.07799 16.987 2.0666C15.676 2.0552 14.413 2.55918 13.47 3.46997L11.75 5.17997M14 11C13.5705 10.4259 13.0226 9.9508 12.3934 9.60704C11.7642 9.26328 11.0684 9.05886 10.3533 9.00765C9.63816 8.95643 8.92037 9.05961 8.24861 9.3102C7.57685 9.56079 6.96684 9.95291 6.45996 10.46L3.45996 13.46C2.54917 14.403 2.04519 15.666 2.05659 16.977C2.06798 18.288 2.59382 19.542 3.52086 20.4691C4.4479 21.3961 5.70197 21.922 7.01295 21.9334C8.32393 21.9447 9.58694 21.4408 10.53 20.53L12.24 18.82" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</a>
<span class="ts-share-notification" data-share-notification>Copied!</span>
</div>
</div>
</div>
</div>
</div>
</aside>
</div>
</div>
<!-- Next and Previous Post Pagination -->
<div style="margin-top: 5rem;">
<div class="ts-container">
<div class="ts-post-pagination">
<div>
<p class="ts-section-heading">Previous Post</p>
<div class="ts-post-unfurl">
<div class="ts-post-unfurl-content">
<div>
<a href="https://slack.engineering/deploy-safety/">
<strong>Deploy Safety: Reducing customer impact from change</strong>
<span aria-label="(opens in new tab)"></span>
</a>
<p class="ts-post-unfurl-text">
It’s mid 2023 and we’ve identified some opportunities to improve our reliability. Fast forward to&hellip; </p>
</div>
<div class="ts-entry__post-meta">
<div class="ts-meta-date">October 7, 2025</div>
<div class="ts-meta-reading-time"><div class="ts-meta-reading-time">11 min read</div></div>
</div>
</div>
<div class="ts-post-unfurl-img">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2025/05/deploy-safety-smaller.jpeg?w=160&amp;h=160&amp;crop=1" class="attachment-thumbnail size-thumbnail wp-post-image" alt="A rocket streaking across the sky wearing a life vest to symbolize deploying safely" decoding="async" loading="lazy" srcset="https://slack.engineering/wp-content/uploads/sites/7/2025/05/deploy-safety-smaller.jpeg?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2025/05/deploy-safety-smaller.jpeg?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2025/05/deploy-safety-smaller.jpeg?resize=110,110 110w" sizes="auto, (max-width: 160px) 100vw, 160px" /> </div>
</div>
</div>
<div>
<p class="ts-section-heading">Next Post</p>
<div class="ts-post-unfurl">
<div class="ts-post-unfurl-content">
<a href="https://slack.engineering/build-better-software-to-build-software-better/">
<strong>Build better software to build software better</strong>
<span aria-label="(opens in new tab)"></span>
</a>
<p class="ts-post-unfurl-text">
We manage the build pipeline that delivers Quip and Slack Canvas’s backend. A year ago,&hellip; </p>
<div class="ts-entry__post-meta">
<div class="ts-meta-date">November 6, 2025</div>
<div class="ts-meta-reading-time"><div class="ts-meta-reading-time">18 min read</div></div>
</div>
</div>
<div class="ts-post-unfurl-img">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2025/10/DevXP-Council-Presentation6-e1759869480617.png?w=160&amp;h=160&amp;crop=1" class="attachment-thumbnail size-thumbnail wp-post-image" alt="An example service build graph. It shows a set of Python files building into a backend artifact and a set of TypeScript files building into a frontend artifact." decoding="async" loading="lazy" srcset="https://slack.engineering/wp-content/uploads/sites/7/2025/10/DevXP-Council-Presentation6-e1759869480617.png?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/DevXP-Council-Presentation6-e1759869480617.png?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2025/10/DevXP-Council-Presentation6-e1759869480617.png?resize=110,110 110w" sizes="auto, (max-width: 160px) 100vw, 160px" /> </div>
</div>
</div>
</div>
</div>
</div>
<div class="ts-entry__post-related">
<div class="ts-container">
<p class="ts-section-heading">Recommended Reading</p>
<div class="ts-entry__post-wrap">
<article class="post-17832 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-migration ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/05/cover_1.jpg?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">July 14, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/"><span>Shipyard: How We Built Slack’s Next-Generation EC2 Platform</span></a></h2><span class="ts-mention">@Archie Gunasekara</span> </div>
</div>
</article>
<article class="post-17812 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-backend tag-cloud-computing tag-collaboration tag-engineering tag-infrastructure tag-innovation tag-machine-learning tag-software-development ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/05/pexels-photograph-6732010.jpg?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">May 28, 2026</div> <div class="ts-meta-reading-time">17 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/"><span>Slack AI: The Path to Multi-Cloud</span></a></h2><span class="ts-mention">@Shaurya Kethireddy</span> </div>
</div>
</article>
<article class="post-17774 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-airflow tag-aws tag-big-data tag-data-engineering tag-data-infrastructure tag-security ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/04/SSH-Rest-CoverImage.png?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="From SSH to REST_ A Security-Driven Modernization of Slack&#039;s EMR Data Pipelines" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">May 5, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/"><span>From SSH to REST: A Security-Driven Modernization of Slack&#8217;s EMR Data Pipelines</span></a></h2><span class="ts-mention">@Mahendran Vasagam</span> </div>
</div>
</article>
<article class="post-16966 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-infrastructure ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/advancing-our-chef-infrastructure/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">September 17, 2024</div> <div class="ts-meta-reading-time">14 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/advancing-our-chef-infrastructure/"><span>Advancing Our Chef Infrastructure</span></a></h2><span class="ts-mention">@Archie Gunasekara</span> </div>
</div>
</article>
</div>
</div>
</div>
<!-- Arc -->
<div class="c-arc concave-up">
<div class="c-arc__upper color-blue"></div>
<div class="c-arc__lower color-white"></div>
</div>
</div>
</main>
<footer class="ts-footer">
<div class="ts-container">
<div class="ts-footer__item">
<div class="ts-footer__col ts-col-left">
<div class="ts-footer__inner">
<div class="ts-logo">
<a class="ts-footer__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
</div>
<div class="ts-footer__col ts-col-canter">
<div class="ts-footer__inner">
<nav class="ts-footer__nav">
<div class="ts-footer__nav-item">
<ul class="ts-footer__nav-inner">
<li><a href="https://slack.engineering/articles/">Articles</a></li>
<!--<li><a href="https://slack.engineering/about/">About</a></li>-->
<li><a href="https://slack.com/careers/dept/software-engineering" target="_blank">Careers</a></li>
<li><a href="https://slack.dev/blog/" target="_blank">Slack Developer Blog</a></li>
<li><a href="https://slack.com/blog/" target="_blank">The Slack Blog</a></li>
</ul>
</div>
</nav>
</div>
</div>
<div class="ts-footer__col ts-col-right">
<div class="ts-footer__inner">
<span>
<a href="https://x.com/SlackEng" target="_blank" class="ts-share-buttons-twitter">
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M9.70487 6.85148L15.289 0.5H13.9657L9.11705 6.0149L5.24443 0.5H0.777832L6.63398 8.8395L0.777832 15.5H2.10116L7.22147 9.67608L11.3112 15.5H15.7778L9.70455 6.85148H9.70487ZM7.89239 8.91297L7.29905 8.08255L2.57797 1.47476H4.61052L8.42048 6.80746L9.01383 7.63788L13.9663 14.5696H11.9338L7.89239 8.91329V8.91297Z" fill="currentcolor"></path>
</svg>
</a>
<a href="https://www.youtube.com/@Slackhq" target="_blank" class="ts-share-buttons-youtube">
<svg width="21" height="14" viewBox="0 0 21 14" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M10.463 0C19.0721 0 20.1483 0 20.1483 7C20.1483 14 19.0721 14 10.463 14C1.85397 14 0.777832 14 0.777832 7C0.777832 0 1.85397 0 10.463 0ZM8.46143 2.7998C8.10631 2.7998 7.77271 3.0798 7.77271 3.4998V10.4998C7.77271 10.9521 8.11707 11.1998 8.46143 11.1998C8.60133 11.1998 8.7197 11.1567 8.82732 11.0921L14.3694 7.59211C14.5846 7.47365 14.6923 7.26904 14.6923 6.9998C14.6923 6.73057 14.5846 6.52596 14.3694 6.4075L8.82732 2.9075C8.70894 2.83211 8.59057 2.7998 8.46143 2.7998Z" fill="currentcolor"></path>
</svg>
</a>
</span>
</div>
</div>
</div>
<br/><br/>
<div class="ts-footer__item">
<p class="ts-footer__info">
<a href="https://slack.com/terms-of-service/user" target="_blank">Terms of Service</a>
<a href="https://slack.com/trust/privacy/privacy-policy" target="_blank">Privacy Information</a>
<a href="#" class="optanon-show-settings">Cookie Preferences</a>
<a href="https://www.salesforce.com/form/other/privacy-request/?_gl=1*1yvb56x*_gcl_au*OTY0OTg5MzcuMTcyNTkxMjE5Nw..*_ga*MTMzNjk3MzQ3MC4xNzE4MDQ0MzEy*_ga_QTJQME5M5D*MTcyNjI2OTEwNy40OS4xLjE3MjYyNjkyMzAuNS4wLjA." target="_blank">Your Privacy Choices</a>
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="30" height="14"><path d="M7.4 12.8h6.8l3.1-11.6H7.4C4.2 1.2 1.6 3.8 1.6 7s2.6 5.8 5.8 5.8z" style="fill-rule:evenodd;clip-rule:evenodd;fill:#fff"/><path d="M22.6 0H7.4c-3.9 0-7 3.1-7 7s3.1 7 7 7h15.2c3.9 0 7-3.1 7-7s-3.2-7-7-7zm-21 7c0-3.2 2.6-5.8 5.8-5.8h9.9l-3.1 11.6H7.4c-3.2 0-5.8-2.6-5.8-5.8z" style="fill-rule:evenodd;clip-rule:evenodd;fill:#06f"/><path d="M24.6 4c.2.2.2.6 0 .8L22.5 7l2.2 2.2c.2.2.2.6 0 .8-.2.2-.6.2-.8 0l-2.2-2.2-2.2 2.2c-.2.2-.6.2-.8 0-.2-.2-.2-.6 0-.8L20.8 7l-2.2-2.2c-.2-.2-.2-.6 0-.8.2-.2.6-.2.8 0l2.2 2.2L23.8 4c.2-.2.6-.2.8 0z" style="fill:#fff"/><path d="M12.7 4.1c.2.2.3.6.1.8L8.6 9.8c-.1.1-.2.2-.3.2-.2.1-.5.1-.7-.1L5.4 7.7c-.2-.2-.2-.6 0-.8.2-.2.6-.2.8 0L8 8.6l3.8-4.5c.2-.2.6-.2.9 0z" style="fill:#06f"/></svg>
<br/><br/>
&copy; 2026 Slack Technologies, LLC, a Salesforce company. All rights reserved. Various trademarks held by their respective owners.
</p>
</div>
</div>
</footer>
</div>
</div>
<a href="#top" data-scroll-to-top class="ts-scroll-to-top">
<svg class="ts-icon-up" width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M5 12H19M19 12L12 5M19 12L12 19" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
<span>scroll to top</span>
</a>
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/sites/7/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/tinyspeck/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script type="text/javascript" src="https://slack.engineering/_static/??-eJydj0sOwjAMRC9EaypVFBaIs4TECg6xE9UuiNvTCFixQSzn80YauNfOFzEUg5qXSKJQZ1J2Rh4ynV8K0tteyzP2STfwO5idxC5pkX+4G/EHI/F5CagtI3YRNRcXMPRMX9N2QV6bRvLQiv4KThVNQWffcCVrL058HKb9tNuOh2FMT0ZXY2k=" ></script><script id="jetpack-stats-js-before">
_stq = window._stq || [];
_stq.push([ "view", {"v":"ext","blog":"195077423","post":"17611","tz":"-7","srv":"slack.engineering","hp":"vip","j":"1:16.1.3"} ]);
_stq.push([ "clickTrackerInit", "195077423", "17611" ]);
//# sourceURL=jetpack-stats-js-before
</script>
<script data-wp-strategy="defer" defer fetchpriority="low" id="jetpack-stats-js" src="https://stats.wp.com/e-202637.js"></script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://slack.engineering/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://slack.engineering/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>