Files
nexus/sreweekly/articles/444/06-advancing-our-chef-infrastructure.html
2026-09-12 17:23:01 +08:00

643 lines
82 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<html lang="en-US">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, height=device-height, initial-scale=1.0, maximum-scale=1">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<link rel="profile" href="https://gmpg.org/xfn/11" />
<link rel="shortcut icon" href="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/icons/favicon.png" sizes="16x16 32x32 48x48" type="image/png">
<!-- Optanon Consent Notice start -->
<script src="https://d34u8crftukxnk.cloudfront.net/onetrust/slack.engineering/production/scripttemplates/otSDKStub.js" type="text/javascript" charset="UTF-8" data-domain-script="019872bf-9ff8-7d80-8158-f0c910bcdfad"></script>
<!-- Optanon Consent Notice end -->
<title>Advancing Our Chef Infrastructure | Engineering at Slack</title>
<meta name="description" content="At Slack, we manage tens of thousands of EC2 instances that host a variety of services, including our Vitess databases, Kubernetes workers, and various components of the Slack application. The majority of these instances run on some version of Ubuntu, while a portion operates on Amazon Linux. With such a vast infrastructure, the critical question&hellip;">
<meta property="og:url" content="https://slack.engineering/advancing-our-chef-infrastructure/">
<meta property="og:type" content="website">
<meta property="og:title" content="Advancing Our Chef Infrastructure">
<meta property="og:description" content="At Slack, we manage tens of thousands of EC2 instances that host a variety of services, including our Vitess databases, Kubernetes workers, and various components of the Slack application. The majority of these instances run on some version of Ubuntu, while a portion operates on Amazon Linux. With such a vast infrastructure, the critical question&hellip;">
<meta property="og:image" content="https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg">
<meta property="og:image:width" content="6000">
<meta property="og:image:height" content="4000">
<meta name="twitter:card" content="summary_large_image">
<meta property="twitter:domain" content="slack.engineering">
<meta property="twitter:url" content="https://slack.engineering/advancing-our-chef-infrastructure/">
<meta name="twitter:title" content="Advancing Our Chef Infrastructure">
<meta name="twitter:description" content="At Slack, we manage tens of thousands of EC2 instances that host a variety of services, including our Vitess databases, Kubernetes workers, and various components of the Slack application. The majority of these instances run on some version of Ubuntu, while a portion operates on Amazon Linux. With such a vast infrastructure, the critical question&hellip;">
<meta name="twitter:image" content="https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg">
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-590Q1VHB01"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-590Q1VHB01');
</script>
<!-- End Google Analytics -->
<title>Advancing Our Chef Infrastructure &#8211; Engineering at Slack</title>
<meta name='robots' content='max-image-preview:large' />
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://slack.engineering/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://slack.engineering/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='all-css-12' href='https://slack.engineering/_static/??-eJxtjEkOgkAQRS8klopGXRjP0hQlVHoMvwjh9ja61OUf3qOlNJyTSTIqYR40gRAc+6196UCwNQhhzJNx7gV7Bnb0hyqTIjpTpqDdN1H9ko0Spcneaa/uh/6s9aRpRRH25ACxapu7UFWbAGqycc/4OF5vbXs53Q/nN25bRHg=' type='text/css' media='all' />
<script type="text/javascript" src="https://slack.engineering/_static/??-eJyFi0sOgCAQQy+kjiRGdGE8i0HEQRiUT4y3FxL3rprXvsJ91sJRlBThNEkhBQhmEUdpN1SgM+/OR+FWGRodKsgPJGFS5rLqK0n/fNFYpF+ptqj8EuUnz3ZifOB9242s0y+jLDUm" ></script><link rel="https://api.w.org/" href="https://slack.engineering/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://slack.engineering/wp-json/wp/v2/posts/16966" /> <style>img#wpstats{display:none}</style>
<link rel="icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=32" sizes="32x32" />
<link rel="icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=192" sizes="192x192" />
<link rel="apple-touch-icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=180" />
<meta name="msapplication-TileImage" content="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=270" />
</head>
<body class="wp-singular post-template-default single single-post postid-16966 single-format-standard wp-embed-responsive wp-theme-tinyspeck">
<a class="ts-skip-link" href="#main">Skip to main content</a>
<div class="ts-site">
<div class="ts-site-inner">
<div class="ts-header-before"></div>
<header class="ts-header">
<div class="ts-container">
<div class="ts-header__inner ts-header__inner-desktop">
<div class="ts-header__col ts-col-left">
<div class="ts-logo">
<a class="ts-header__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
<div class="ts-header__col ts-col-center"></div>
<div class="ts-header__col ts-col-right">
<!--<a href="https://slack.engineering/articles/">Articles</a>
<a href="https://slack.engineering/about/">About</a>-->
<span class="ts-header__search-toggle" role="button" data-search-toggle="open">
<span class="ts-header__search-toggle-label">Search</span>
<svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</span>
</div>
</div>
<div class="ts-header__inner ts-header__inner-mobile">
<div class="ts-header__col ts-col-left"></div>
<div class="ts-header__col ts-col-center">
<div class="ts-logo">
<a class="ts-header__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
<div class="ts-header__col ts-col-right">
<span class="ts-header__search-toggle" role="button" data-search-toggle="open">
<span class="ts-header__search-toggle-label">Search</span>
<svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</span>
</div>
</div>
</div>
<div class="ts-search">
<div class="ts-container">
<div class="ts-search__close">
<div class="ts-search__close-inner" role="button" data-search-toggle="close">
<span class="ts-search__close-label">Close</span>
<svg class="ts-search__close-icon ts-icon ts-icon-x" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M18 6L6 18M6 6L18 18" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</div>
</div>
<div class="ts-search__wrapper">
<form role="search" method="get" class="ts-search__nav-form" action="https://slack.engineering/">
<div class="ts-search__group">
<input class="ts-search__input" type="search" required value="" name="s" placeholder="What are you looking for?">
<button class="ts-search__submit">Search</button>
</div>
</form>
</div>
</div>
</div>
</header>
<main id="main" class="ts-site-primary">
<div class="ts-site-content ts-sidebar-enabled ts-singular-thumbnail-enabled">
<div class="ts-container">
<div class="ts-main-content">
<div class="ts-content-area">
<div class="ts-entry__header ts-entry__header-large">
<div class="ts-entry__header-inner">
<div class="ts-entry__header-info">
<div class="ts-entry__post-meta-wrapper">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">September 17, 2024</div>
<div class="ts-meta-reading-time">14 min read</div> </div>
</div>
<h1 class="ts-jumbo"><span>Advancing Our Chef Infrastructure</span></h1> <p class="ts-hero-paragraph">Enhancing Safety and Scalability</p>
<div class="c-article__author__meta"><div class="c-article__author"><div class="c-article__author__avatar"><img src="https://slack.engineering/wp-content/uploads/sites/7/2020/09/Screenshot-2023-03-01-at-9.02.01-am.png"/></div><div class="c-article__author__info"><span class="c-article__author__name">Archie Gunasekara</span><span class="c-article__author__title">Staff Software Engineer, Cloud</span></div></div></div> </div>
<figure class="ts-entry__post-media post-media filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg?w=1020" class="attachment-post-thumbnail size-post-thumbnail wp-post-image" alt="" decoding="async" fetchpriority="high" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg 6000w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg?resize=640,427 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg?resize=768,512 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg?resize=1280,853 1280w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg?resize=1536,1024 1536w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg?resize=2048,1365 2048w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg?resize=380,253 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg?resize=800,533 800w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/myles-tan-IWCljYv1TJw-unsplash.jpg?resize=1160,773 1160w" sizes="(max-width: 1020px) 100vw, 1020px" /> </figure>
</div>
</div>
<div class="ts-entry__wrap">
<div class="ts-entry__container">
<aside class="ts-widget-area ts-sidebar__area">
<div class="ts-sidebar__inner">
<!-- Search -->
<div class="widget widget_search">
<p class="ts-section-heading">Search</p>
<form role="search" method="get" class="ts-search__form" action="https://slack.engineering/">
<div class="ts-search__container">
<input required class="ts-search__input" type="search" value="" name="s" placeholder="">
<button class="ts-search__submit"><svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg></button>
</div>
</form>
</div>
<!-- Latest Posts -->
<div class="widget">
<p class="ts-section-heading">Latest Posts</p>
<div class="widget-body ts-widget-posts">
<ul>
<li class="ts-post-item">
<article class="post-17832 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-migration ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">July 14, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/">Shipyard: How We Built Slack’s Next-Generation EC2 Platform</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17854 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-automation-testing tag-developer-productivity tag-innovation ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">June 11, 2026</div> <div class="ts-meta-reading-time">12 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/agentic-testing-where-agents-fit-in-the-e2e-testing-stack/">Agentic Testing: Where Agents Fit in the E2E Testing Stack</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17812 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-backend tag-cloud-computing tag-collaboration tag-engineering tag-infrastructure tag-innovation tag-machine-learning tag-software-development ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">May 28, 2026</div> <div class="ts-meta-reading-time">17 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/">Slack AI: The Path to Multi-Cloud</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17774 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-airflow tag-aws tag-big-data tag-data-engineering tag-data-infrastructure tag-security ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">May 5, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/">From SSH to REST: A Security-Driven Modernization of Slack&#8217;s EMR Data Pipelines</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17752 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-development tag-security tag-software-engineering ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">April 13, 2026</div> <div class="ts-meta-reading-time">14 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/managing-context-in-long-run-agentic-applications/">Managing context in long-run agentic applications</a>
</h2>
</div>
</div>
</article>
</li>
</ul>
</div>
</div>
<!-- Archives -->
<div class="widget">
<p class="ts-section-heading">Archives</p>
<ul class="wp-block-archives-list wp-block-archives">
<li><a href='https://slack.engineering/2026/'>2026</a>&nbsp;(7)</li>
<li><a href='https://slack.engineering/2025/'>2025</a>&nbsp;(9)</li>
<li><a href='https://slack.engineering/2024/'>2024</a>&nbsp;(19)</li>
<li><a href='https://slack.engineering/2023/'>2023</a>&nbsp;(16)</li>
<li><a href='https://slack.engineering/2022/'>2022</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2021/'>2021</a>&nbsp;(24)</li>
<li><a href='https://slack.engineering/2020/'>2020</a>&nbsp;(26)</li>
<li><a href='https://slack.engineering/2019/'>2019</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2018/'>2018</a>&nbsp;(11)</li>
<li><a href='https://slack.engineering/2017/'>2017</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2016/'>2016</a>&nbsp;(19)</li>
</ul>
</div>
</div>
</aside>
<div class="ts-entry__content-wrap">
<div class="ts-single-post ts-wysiwyg">
<p>At Slack, we manage tens of thousands of EC2 instances that host a variety of services, including our Vitess databases, Kubernetes workers, and various components of the Slack application. The majority of these instances run on some version of Ubuntu, while a portion operates on Amazon Linux. With such a vast infrastructure, the critical question arises: how do we efficiently provision these instances and deploy changes across them? The solution lies in a combination of internally-developed services, with Chef playing a central role. In this blog post, I’ll discuss the evolution of our Chef infrastructure over the years and the challenges we encountered along the way.</p>
<h2>A journey down memory lane: our previous process</h2>
<p>In the early days of Slack, we relied on a single Chef stack. Our Chef infrastructure included a set of EC2 instances, an AWS application load balancer, an RDS cluster, and an AWS OpenSearch cluster. As our fleet of instances started to grow, we scaled up and out this stack to match our increasing demand.</p>
<p>We had three environments on this Chef stack: Sandbox, Dev, and Prod. Nodes from each environment in our fleet were mapped to one of these environments. Slack has two types of cookbooks: cookbooks we download from the Chef Supermarket and the ones that are internally created. When these cookbooks are modified, we upload them using a process called DishPig. When a change is merged to our repo, we triggered a CI job to look for any changes to the cookbooks or roles and built an artifact with this. Then this artifact gets uploaded to a S3 bucket and then from this S3 bucket, we notify an SQS queue. We triggered the DishPig process on the top of each hour to look for any new messages in the queue. When DishPig detected a new event in the queue, it downloaded the artifact and uploaded the changed cookbooks to the Chef server. We upload the cookbooks we take from the supermarket with their own version, but we will always upload the cookbooks we create with a fixed version number. Therefore we did not have multiple versions of the cookbooks we created and only a single version of them were available on the Chef server.</p>
<p>Once the cookbooks were uploaded to the Chef server, all three environments were updated to include the latest versions of the Supermarket cookbooks. Since we are uploading our own cookbooks with the same version, their references in the environments did not need updating.</p>
<p>With this approach, all our environments got the changes on the hour.</p>
<p><img decoding="async" width="884" height="94" class="alignnone size-medium wp-image-16993" src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_1_9a3875.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_1_9a3875.png 884w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_1_9a3875.png?resize=640,68 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_1_9a3875.png?resize=768,82 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_1_9a3875.png?resize=380,40 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_1_9a3875.png?resize=800,85 800w" sizes="(max-width: 884px) 100vw, 884px" /></p>
<p>We also provided tooling that allowed developers to create custom Chef environments with specific versions of cookbooks, enabling them to test their changes on a small set of nodes within a temporary environment. Once testing was complete, the tooling would automatically clean up the temporary environment and remove the custom cookbook versions.</p>
<p>This approach had a few significant drawbacks: all changes were deployed across all environments simultaneously. This meant that any faulty changes could potentially disrupt all existing Chef runs and new server provisions across our entire infrastructure.</p>
<p>Relying on a single Chef stack also meant that any issues with this stack could have widespread effects, potentially impacting our entire infrastructure.</p>
<h2>Transitioning to a sharded Chef infrastructure</h2>
<p>As our infrastructure grew rapidly, ensuring guaranteed reliability became a top priority. We recognized that relying on a single Chef stack posed a significant risk, as it was a major single point of failure. To address this, we began redesigning our Chef infrastructure, starting by eliminating our dependence on a single Chef stack.</p>
<p>We decided to create multiple Chef stacks to distribute the load more effectively and ensure resilience. This approach also gave us the flexibility to direct new provisions to the remaining operational Chef stacks if one were to fail. However, implementing this new strategy introduced several challenges. Let’s explore the most significant ones.</p>
<h3>Challenge 1: Assigning a shard to a node</h3>
<p>Our first step was to find a method for directing new provisions to specific shards. We achieved this using an AWS Route53 Weighted CNAME record. When an instance spins up, it queries the CNAME record and, based on the assigned weight, receives a record from the set. This record then determines which Chef stack the instance will be assigned to.</p>
<p>Additionally, as illustrated below, we separated the development and production Chef infrastructure into distinct stacks. This segregation helps strengthen the boundary between our development and production environments.</p>
<p><img decoding="async" width="1325" height="209" class="alignnone size-medium wp-image-16968" src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/1_cnames.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/1_cnames.png 1325w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/1_cnames.png?resize=640,101 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/1_cnames.png?resize=768,121 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/1_cnames.png?resize=1280,202 1280w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/1_cnames.png?resize=380,60 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/1_cnames.png?resize=800,126 800w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/1_cnames.png?resize=1160,183 1160w" sizes="(max-width: 1325px) 100vw, 1325px" /></p>
<h3>Challenge 2: Neighborhood discovery</h3>
<p>Historically, Slack lacked a dedicated EC2 inventory management system, and our Chef stack was the closest alternative. As a result, teams began querying the Chef server whenever they needed information about specific instances in the fleet. They would use Chef search to locate nodes based on criteria, such as nodes with a particular Chef role in a specific AWS region.</p>
<p>This approach worked well until we moved to a sharded Chef infrastructure. With the new setup, running the same query would only return nodes from the specific Chef stack you queried. Since nodes are now distributed across multiple Chef stacks, this method no longer provides a complete view of all instances.</p>
<p>Let’s take a step back and look at how a Chef run operates. A Chef run consists of two phases: compile and converge. During the compile phase, Chef collects information about its run lists, reads all its cookbooks, assigns variables, creates attributes, and maps out the resources it needs to create. In the converge phase, Chef then creates those resources based on the information gathered in the compile phase.</p>
<p>Some of our cookbooks were designed to create attributes and variables based on the results from Chef searches. With the transition to sharded Chef stacks, we needed to find an alternative to these node discovery methods. After considering our options, we decided to leverage an existing service at Slack: Consul.</p>
<p>We began registering certain services in Consul for service discovery, taking advantage of Consul’s tagging capabilities. By creating and registering services, we could now query Consul for information about other nodes, replacing the need for Chef searches.</p>
<p>However, this introduced a new challenge. At Slack, we use an overlay network called <a href="https://github.com/slackhq/nebula">Nebula</a>, which transparently handles network encryption between endpoints. To query Consul, instances need to be connected through Nebula. Yet, Nebula is configured by Chef, creating a circular dependency problem. We couldn’t set up attributes or variables using Consul during Chef’s compile phase, leading to a classic chicken-and-egg situation.</p>
<p>After some consideration, we devised a solution using Chef&#8217;s <a href="https://docs.chef.io/resources/ruby_block/">ruby_block</a> resources. We placed the logic for creating resources and assigning variables within these <a href="https://docs.chef.io/resources/ruby_block/">ruby_block</a> resources. Since <a href="https://docs.chef.io/resources/ruby_block/">ruby_block</a> resources are executed only during the converge phase of the Chef run, we can control the execution order to ensure that Nebula is installed and configured before these resources are processed.</p>
<p>However, for values calculated within Ruby blocks that need to be used elsewhere, we utilize the lazy keyword. This ensures that these values are loaded during the Chef converge phase.</p>
<p>We also developed a set of Chef library functions to facilitate node lookups based on various tags, service names, and other criteria. This allowed us to replicate the functionality we previously achieved with Chef search. We integrated these helper functions into our cookbooks, enabling us to retrieve the necessary values from Consul efficiently.</p>
<p><span style="font-weight: 400">Example:</span></p>
<pre><code class="language-ruby">node.override[&#039;some_attribute&#039;] = []
# In Chef, when a resource is defined all its variables are evaluated during
# compile time and the execution of the resource takes place in converge phase.
# So if the value of a particular attribute is changed in converge
# (and not in compile) the resource will be executed with the old value.
# Therefore we need to put the following call inside the ruby block because,
# We need to make sure Nebula is up so we can connect to Consul (Nebula gets stup during converge time)
ruby_block &#039;lets_set_some_attribute&#039; do
block do
extend SomeHelper
node.override[&#039;some_attribute&#039;] = consul_service_lookup_helper_func()
end
end
# Please note that we are not using `lazy` with the `only_if` below for the value that is calculated in the ruby block above
# It&#039;s because `only_if` is already lazy
# https://github.com/chef/chef/issues/10243#issuecomment-668187830
systemd_unit &#039;some_service&#039; do
action [:enable, :start]
only_if { node.override[&#039;some_attribute&#039;].empty?
end</code></pre>
<p>Although this approach required adding large Ruby blocks with complex logic to some cookbooks, it allowed us to efficiently roll out multiple Chef stacks. If you&#8217;re a Chef expert with suggestions for a better method, we&#8217;d love to hear from you. Please don&#8217;t hesitate to reach out!</p>
<h3>Challenge 3: Searching Chef</h3>
<p>As mentioned earlier, we initially relied on Chef as an inventory management system. This approach was useful for discovering information about nodes and their attributes within our fleet. For instance, we could track the progress of feature rollouts using these attributes. To facilitate Chef searches and reporting for developers, we created an interface called Gaz.</p>
<p><img loading="lazy" decoding="async" width="1916" height="412" class="alignnone size-medium wp-image-16970" src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/2_gaz.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/2_gaz.png 1916w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/2_gaz.png?resize=640,138 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/2_gaz.png?resize=768,165 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/2_gaz.png?resize=1280,275 1280w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/2_gaz.png?resize=1536,330 1536w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/2_gaz.png?resize=380,82 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/2_gaz.png?resize=800,172 800w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/2_gaz.png?resize=1160,249 1160w" sizes="auto, (max-width: 1916px) 100vw, 1916px" /></p>
<p>This approach worked well with a single Chef stack, but with multiple stacks in play, we needed a new solution. We developed a service called Shearch (Sharded Chef Search) which features an API that accepts Chef queries, runs them across multiple shards, and consolidates the results. This allows us to use Shearch behind Gaz, rather than having Gaz interact directly with a Chef stack.</p>
<p>Additionally, we replaced the Knife command on our development boxes with a new tool called Gnife (a nod to &#8220;Go Knife,&#8221; as it’s written in Go). Gnife offers the same functionality as the Chef Knife command but operates across multiple shards using the Shearch service.<br />
We also had to update many of our internally developed tools and libraries to use the Shearch service instead of interacting directly with a Chef stack. As you can imagine, this required considerable effort.</p>
<p>Sharding our Chef infrastructure significantly enhanced its resilience. However, we still faced challenges with how we deploy our cookbooks. Currently, we update all our environments simultaneously and upload the same cookbook versions across the board. We needed a more efficient approach to improve this process.</p>
<h3>Challenge 4: Cookbook uploads</h3>
<p>To implement this change quickly, we decided to retain our existing DishPig cookbook uploader application. However, since DishPig was designed to work with a single Chef stack, we had to deploy multiple instances of the application, each dedicated to a specific stack.</p>
<p>We modified our S3 bucket to send notifications to an SNS topic (instead of directly to SQS), which then fanned out these messages to multiple SQS queues, each corresponding to a different DishPig deployment. This setup allowed changes to be uploaded independently to each Chef stack without any consistency checks between them.</p>
<p>For example, we encountered issues where artifacts built close to the top of the hour (when DishPig is triggered) resulted in SQS messages reaching some queues on time but not others. This led to Chef stacks having different versions of cookbooks. While we did create a small monitoring system to check for and alert on consistency errors, it was far from an ideal solution.</p>
<p><img loading="lazy" decoding="async" width="1084" height="379" class="alignnone size-medium wp-image-16995" src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_2_03599a.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_2_03599a.png 1084w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_2_03599a.png?resize=640,224 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_2_03599a.png?resize=768,269 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_2_03599a.png?resize=380,133 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/dishpig_2_03599a.png?resize=800,280 800w" sizes="auto, (max-width: 1084px) 100vw, 1084px" /></p>
<h2>Cookbook versioning and Chef Librarian</h2>
<p>We needed a solution that would allow us to version our cookbooks and update environments independently, while also managing multiple Chef stacks and ensuring they remain in sync. Additionally, it was crucial for the application to track what changes are included in each version and where it has been deployed. To address these requirements, we designed a new service called Chef Librarian to replace our previous service, DishPig, as discussed earlier.</p>
<p>Our cookbooks are managed within a single Git repository. Internally developed cookbooks are stored in the site-cookbooks directory, while Supermarket cookbooks are kept in the cookbooks directory.</p>
<pre><code class="language-">.
├── cookbooks
│ ├── supermarket-cookbook-1
│ │ ├── files
│ │ │ └── default
│ │ │ └── file1.txt
│ │ └── recipes
│ │ └── default.rb
│ └── supermarket-cookbook-2
│ ├── files
│ │ └── default
│ │ └── file1.txt
│ └── recipes
│ └── default.rb
├── site-cookbooks
│ ├── our-cookbook-1
│ │ ├── files
│ │ │ └── default
│ │ │ └── file1.txt
│ │ └── recipes
│ │ └── default.rb
│ └── our-cookbook-2
│ ├── files
│ │ └── default
│ │ └── file1.txt
│ └── recipes
│ └── default.rb
└── roles
├── sample-role-1.json
└── sample-role-2.json</code><code class="language-vim">
</code></pre>
<p>When changes to any of these items are merged, we use GitHub Actions to trigger a build job. This job creates a tarball that includes a complete copy of the repository. For internally developed cookbooks, we update the version number in the site-cookbooks directory to a new version using the standard Chef cookbook versioning format: <code>YYYYMMDD.TIMESTAMP.0</code> (e.g., <code>20240809.1723164435.0</code>). The version numbers for Supermarket cookbooks remain unchanged unless we fetch a new version from the upstream source.</p>
<p>Once the artifact is built, GitHub Actions uploads it to S3. S3 then generates a notification in an SQS queue, which the Chef Librarian application listens to. Chef Librarian promptly processes the new artifact and uploads the updated versions of cookbooks to all our Chef stacks. Although these new versions are now on the Chef stacks, they won’t be utilized until explicitly specified.</p>
<p>With this approach, we upload all cookbooks, regardless of whether they have changed or not, to the Chef stacks with the new version.</p>
<p>We have developed two API endpoints for the Chef Librarian application:</p>
<ul>
<li><code>/update_environment_to_version</code> &#8211; This endpoint updates an environment to a specified version. To use it, send a POST request with parameters including the target environment and the version to update to.</li>
<li><code>/update_environment_from_environment</code> &#8211; This endpoint updates an environment to match the versions of another environment.</li>
</ul>
<p>We now have the ability to update environments to specific versions independently of each other. This enables us to first update the sandbox and development environments, monitor metrics for any potential issues, and ensure stability before promoting the artifact to production.</p>
<p>In practice, this means that any problematic changes deployed to Chef do not impact all environments simultaneously. We can detect and address errors earlier, preventing them from propagating further.</p>
<p>Chef Librarian also stores artifact versions, the environments they’re deployed to, and other state-related information in DynamoDB. This makes it much easier for us to track changes and visualize our rollout process.</p>
<p><img loading="lazy" decoding="async" width="971" height="428" class="alignnone size-medium wp-image-16996" src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_1_48cfc0.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_1_48cfc0.png 971w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_1_48cfc0.png?resize=640,282 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_1_48cfc0.png?resize=768,339 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_1_48cfc0.png?resize=380,167 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_1_48cfc0.png?resize=800,353 800w" sizes="auto, (max-width: 971px) 100vw, 971px" /></p>
<p>However, Chef roles are a different challenge since they aren’t versioned. When you upload them, they propagate across all environments, making role updates a risky operation. To address this, we&#8217;ve stripped most of the logic out of our Chef roles, leaving them with only basic information and a run list. Additionally, because we’ve split our Chef infrastructure into dev and prod environments, we only upload roles to the relevant Chef stacks when their corresponding environments are updated (e.g., roles are uploaded to prod shards only when the prod environment is updated).</p>
<p><img loading="lazy" decoding="async" width="812" height="327" class="alignnone size-medium wp-image-16973" src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_2.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_2.png 812w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_2.png?resize=640,258 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_2.png?resize=768,309 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_2.png?resize=380,153 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_2.png?resize=800,322 800w" sizes="auto, (max-width: 812px) 100vw, 812px" /></p>
<p>Currently, we use a Kubernetes CronJob to call the API endpoints discussed earlier on a schedule, promoting versions across our environments. However, this process first checks for a key in a designated S3 bucket to ensure that promotions are not blocked before proceeding. This means that if we detect failures in the sandbox or dev environments, we can place this key in the bucket to prevent further promotion of artifact versions.</p>
<p>At present, this is a manual process triggered by a human when they receive alerts about Chef run failures or provisioning issues. Our goal, however, is to replace this Kubernetes CronJob with a more intelligent system that can automatically monitor metrics and impose these blocks without human intervention.</p>
<p><img loading="lazy" decoding="async" width="805" height="315" class="alignnone size-medium wp-image-16974" src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_3.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_3.png 805w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_3.png?resize=640,250 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_3.png?resize=768,301 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_3.png?resize=380,149 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Chef_Librarian_3.png?resize=800,313 800w" sizes="auto, (max-width: 805px) 100vw, 805px" /></p>
<p>We’ve also developed an interface for Chef Librarian, allowing users to view which artifacts have been uploaded with specific commits, as well as when an environment was updated to a particular version in a specific Chef shard.</p>
<p><img loading="lazy" decoding="async" width="2028" height="629" class="alignnone size-medium wp-image-16977" src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/Librarian_UI_66f25d.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/Librarian_UI_66f25d.png 2028w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Librarian_UI_66f25d.png?resize=640,199 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Librarian_UI_66f25d.png?resize=768,238 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Librarian_UI_66f25d.png?resize=1280,397 1280w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Librarian_UI_66f25d.png?resize=1536,476 1536w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Librarian_UI_66f25d.png?resize=380,118 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Librarian_UI_66f25d.png?resize=800,248 800w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/Librarian_UI_66f25d.png?resize=1160,360 1160w" sizes="auto, (max-width: 2028px) 100vw, 2028px" /></p>
<p>We developed a Slack app for the Chef Librarian application, allowing us to notify users when their changes are being promoted to an environment. The app identifies the user who made the change by examining the Git commit and then uses the Slack API to look up their Slack handle. This enables us to tag the user directly in the notification, ensuring they are promptly informed when the message is posted.</p>
<p><img loading="lazy" decoding="async" width="768" height="164" class="alignnone size-medium wp-image-16976" src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/slack_message.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/slack_message.png 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/slack_message.png?resize=640,137 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/slack_message.png?resize=380,81 380w" sizes="auto, (max-width: 768px) 100vw, 768px" /></p>
<h2>What’s next?</h2>
<p>Although we’ve made significant strides in improving deployment safety, there&#8217;s still much more we can do. One avenue we&#8217;re exploring is further segmentation of our Chef environments. For example, we&#8217;re considering breaking down our production Chef environments by AWS availability zones, allowing us to promote changes independently within each zone and prevent bad changes from being deployed everywhere simultaneously.</p>
<p>Another long-term option we’re investigating is the adoption of Chef PolicyFiles and PolicyGroups. While this represents a significant departure from our current setup, it would offer much greater flexibility and safety when deploying changes to specific nodes.</p>
<p>Implementing these changes at our scale is a complex undertaking. We’re still conducting research and assessing the potential impacts and benefits. There’s a lot of exciting work happening in this space at Slack, so stay tuned to our blog for more updates on the cool projects we’re working on!</p>
</div>
<div class="ts-tags-section"><span class="ts-tag">#<a href="https://slack.engineering/tags/aws/" rel="tag">aws</a></span><span class="ts-tag">#<a href="https://slack.engineering/tags/infrastructure/" rel="tag">infrastructure</a></span></div> </div>
</div>
</div>
</div>
<aside class="ts-metabar__area">
<div class="ts-metabar__inner">
<div class="ts-metabar__item">
<div class="ts-share-buttons-wrap">
<div class="ts-share-buttons-items">
<div class="ts-share-buttons-item ts-share-buttons-twitter">
<a href="https://x.com/intent/post?url=https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-twitter" width="21" viewBox="0 0 72 65" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M22.6891 0H0L26.9014 35.3477L1.72089 65H13.355L32.3999 42.5731L49.3109 64.794H72L44.317 28.4191L44.366 28.4818L68.2015 0.412927H56.5674L38.8667 21.2573L22.6891 0ZM12.524 6.19052H19.5874L59.476 58.603H52.4126L12.524 6.19052Z" fill="currentColor"/></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-linkedin">
<a href="https://www.linkedin.com/shareArticle?mini=true&#038;url=https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-linkedin" width="21" height="21" viewBox="0 0 70 70" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M15.8066 23.0188H1.17969V69.663H15.8066V23.0188Z" fill="currentColor"/><path d="M55.3732 22.0077C54.834 21.9403 54.2611 21.9066 53.6881 21.8729C45.4984 21.5359 40.8812 26.389 39.2635 28.4786C38.8254 29.0515 38.6231 29.3886 38.6231 29.3886V23.1536H24.6366V69.7978H38.6231H39.2635C39.2635 65.0458 39.2635 60.3274 39.2635 55.5754C39.2635 53.014 39.2635 50.4526 39.2635 47.8912C39.2635 44.7232 39.0276 41.3529 40.6116 38.4545C41.9597 36.0279 44.3862 34.8147 47.1161 34.8147C55.2047 34.8147 55.3732 42.1281 55.3732 42.8021C55.3732 42.8358 55.3732 42.8695 55.3732 42.8695V70H70.0001V39.5667C70.0001 29.1526 64.7088 23.0188 55.3732 22.0077Z" fill="currentColor"/><path d="M8.49301 16.986C13.1836 16.986 16.9861 13.1836 16.9861 8.49303C16.9861 3.80246 13.1836 0 8.49301 0C3.80244 0 0 3.80246 0 8.49303C0 13.1836 3.80244 16.986 8.49301 16.986Z" fill="currentColor"/></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-facebook">
<a href="https://www.facebook.com/sharer/sharer.php?u=https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-facebook" width="21" height="21" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><g clip-path="url(#clip0_11_112)"><path d="M12 0C18.6274 0 24 5.37259 24 12C24 18.1352 19.3955 23.1944 13.4538 23.9121V15.667L16.7001 15.667L17.3734 12H13.4538V10.7031C13.4538 9.73417 13.6439 9.06339 14.0799 8.63483C14.5159 8.20627 15.1979 8.01993 16.1817 8.01993C16.4307 8.01993 16.6599 8.02241 16.8633 8.02736C17.1591 8.03456 17.4002 8.047 17.568 8.06467V4.74048C17.501 4.72184 17.4218 4.70321 17.3331 4.68486C17.1321 4.6433 16.8822 4.60324 16.6136 4.56806C16.0523 4.49453 15.4093 4.4423 14.9594 4.4423C13.1424 4.4423 11.7692 4.83102 10.8107 5.63619C9.65388 6.60791 9.10108 8.18622 9.10108 10.4199V12H6.62659V15.667H9.10108V23.6466C3.87432 22.3498 0 17.6277 0 12C0 5.37259 5.37259 0 12 0Z" fill="currentColor"/></g><defs><clipPath id="clip0_11_112"><rect width="24" height="24" fill="white"/></clipPath></defs></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-threads">
<a href="https://www.threads.net/intent/post?text=+https%3A%2F%2Fslack.engineering%2Fadvancing-our-chef-infrastructure%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon" width="21" viewBox="0 0 448 512" xmlns="http://www.w3.org/2000/svg"><path d="M331.5 235.7c2.2 .9 4.2 1.9 6.3 2.8c29.2 14.1 50.6 35.2 61.8 61.4c15.7 36.5 17.2 95.8-30.3 143.2c-36.2 36.2-80.3 52.5-142.6 53h-.3c-70.2-.5-124.1-24.1-160.4-70.2c-32.3-41-48.9-98.1-49.5-169.6V256v-.2C17 184.3 33.6 127.2 65.9 86.2C102.2 40.1 156.2 16.5 226.4 16h.3c70.3 .5 124.9 24 162.3 69.9c18.4 22.7 32 50 40.6 81.7l-40.4 10.8c-7.1-25.8-17.8-47.8-32.2-65.4c-29.2-35.8-73-54.2-130.5-54.6c-57 .5-100.1 18.8-128.2 54.4C72.1 146.1 58.5 194.3 58 256c.5 61.7 14.1 109.9 40.3 143.3c28 35.6 71.2 53.9 128.2 54.4c51.4-.4 85.4-12.6 113.7-40.9c32.3-32.2 31.7-71.8 21.4-95.9c-6.1-14.2-17.1-26-31.9-34.9c-3.7 26.9-11.8 48.3-24.7 64.8c-17.1 21.8-41.4 33.6-72.7 35.3c-23.6 1.3-46.3-4.4-63.9-16c-20.8-13.8-33-34.8-34.3-59.3c-2.5-48.3 35.7-83 95.2-86.4c21.1-1.2 40.9-.3 59.2 2.8c-2.4-14.8-7.3-26.6-14.6-35.2c-10-11.7-25.6-17.7-46.2-17.8H227c-16.6 0-39 4.6-53.3 26.3l-34.4-23.6c19.2-29.1 50.3-45.1 87.8-45.1h.8c62.6 .4 99.9 39.5 103.7 107.7l-.2 .2zm-156 68.8c1.3 25.1 28.4 36.8 54.6 35.3c25.6-1.4 54.6-11.4 59.5-73.2c-13.2-2.9-27.8-4.4-43.4-4.4c-4.8 0-9.6 .1-14.4 .4c-42.9 2.4-57.2 23.2-56.2 41.8l-.1 .1z"></path></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-copy">
<a href="#" class="ts-share-buttons-link" data-share-link title="Copy link">
<svg width="21" height="21" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M10 13C10.4295 13.5741 10.9774 14.0491 11.6066 14.3929C12.2357 14.7367 12.9315 14.9411 13.6467 14.9923C14.3618 15.0435 15.0796 14.9403 15.7513 14.6897C16.4231 14.4392 17.0331 14.047 17.54 13.54L20.54 10.54C21.4508 9.59695 21.9548 8.33394 21.9434 7.02296C21.932 5.71198 21.4061 4.45791 20.4791 3.53087C19.5521 2.60383 18.298 2.07799 16.987 2.0666C15.676 2.0552 14.413 2.55918 13.47 3.46997L11.75 5.17997M14 11C13.5705 10.4259 13.0226 9.9508 12.3934 9.60704C11.7642 9.26328 11.0684 9.05886 10.3533 9.00765C9.63816 8.95643 8.92037 9.05961 8.24861 9.3102C7.57685 9.56079 6.96684 9.95291 6.45996 10.46L3.45996 13.46C2.54917 14.403 2.04519 15.666 2.05659 16.977C2.06798 18.288 2.59382 19.542 3.52086 20.4691C4.4479 21.3961 5.70197 21.922 7.01295 21.9334C8.32393 21.9447 9.58694 21.4408 10.53 20.53L12.24 18.82" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</a>
<span class="ts-share-notification" data-share-notification>Copied!</span>
</div>
</div>
</div>
</div>
</div>
</aside>
</div>
</div>
<!-- Next and Previous Post Pagination -->
<div style="margin-top: 5rem;">
<div class="ts-container">
<div class="ts-post-pagination">
<div>
<p class="ts-section-heading">Previous Post</p>
<div class="ts-post-unfurl">
<div class="ts-post-unfurl-content">
<div>
<a href="https://slack.engineering/engineering-with-empathy-my-journey-to-understanding-the-user-experience/">
<strong>Engineering with Empathy: My Journey to Understanding the User Experience</strong>
<span aria-label="(opens in new tab)"></span>
</a>
<p class="ts-post-unfurl-text">
&#8220;What are your goals for this quarter?&#8221; It&#8217;s the question every manager asks, and one&hellip; </p>
</div>
<div class="ts-entry__post-meta">
<div class="ts-meta-date">August 30, 2024</div>
<div class="ts-meta-reading-time"><div class="ts-meta-reading-time">8 min read</div></div>
</div>
</div>
<div class="ts-post-unfurl-img">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2024/08/featured_image.png?w=160&amp;h=160&amp;crop=1" class="attachment-thumbnail size-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/08/featured_image.png?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2024/08/featured_image.png?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2024/08/featured_image.png?resize=110,110 110w" sizes="auto, (max-width: 160px) 100vw, 160px" /> </div>
</div>
</div>
<div>
<p class="ts-section-heading">Next Post</p>
<div class="ts-post-unfurl">
<div class="ts-post-unfurl-content">
<a href="https://slack.engineering/driving-a-project-intern-edition/">
<strong>Driving a Project: Intern Edition</strong>
<span aria-label="(opens in new tab)"></span>
</a>
<p class="ts-post-unfurl-text">
After a lot of hard work, you&#8217;ve landed that coveted internship. Now comes the next&hellip; </p>
<div class="ts-entry__post-meta">
<div class="ts-meta-date">September 23, 2024</div>
<div class="ts-meta-reading-time"><div class="ts-meta-reading-time">7 min read</div></div>
</div>
</div>
<div class="ts-post-unfurl-img">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/caleb-whiting-bau67pJBPcA-unsplash.jpg?w=160&amp;h=160&amp;crop=1" class="attachment-thumbnail size-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/caleb-whiting-bau67pJBPcA-unsplash.jpg?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/caleb-whiting-bau67pJBPcA-unsplash.jpg?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/caleb-whiting-bau67pJBPcA-unsplash.jpg?resize=110,110 110w" sizes="auto, (max-width: 160px) 100vw, 160px" /> </div>
</div>
</div>
</div>
</div>
</div>
<div class="ts-entry__post-related">
<div class="ts-container">
<p class="ts-section-heading">Recommended Reading</p>
<div class="ts-entry__post-wrap">
<article class="post-17832 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-migration ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/05/cover_1.jpg?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">July 14, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/"><span>Shipyard: How We Built Slack’s Next-Generation EC2 Platform</span></a></h2><span class="ts-mention">@Archie Gunasekara</span> </div>
</div>
</article>
<article class="post-17812 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-backend tag-cloud-computing tag-collaboration tag-engineering tag-infrastructure tag-innovation tag-machine-learning tag-software-development ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/05/pexels-photograph-6732010.jpg?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">May 28, 2026</div> <div class="ts-meta-reading-time">17 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/"><span>Slack AI: The Path to Multi-Cloud</span></a></h2><span class="ts-mention">@Shaurya Kethireddy</span> </div>
</div>
</article>
<article class="post-17774 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-airflow tag-aws tag-big-data tag-data-engineering tag-data-infrastructure tag-security ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/04/SSH-Rest-CoverImage.png?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="From SSH to REST_ A Security-Driven Modernization of Slack&#039;s EMR Data Pipelines" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">May 5, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/"><span>From SSH to REST: A Security-Driven Modernization of Slack&#8217;s EMR Data Pipelines</span></a></h2><span class="ts-mention">@Mahendran Vasagam</span> </div>
</div>
</article>
<article class="post-17732 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-golang tag-infrastructure tag-networking tag-observability tag-open-source ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/03/cover.png?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/from-custom-to-open-scalable-network-probing-and-http-3-readiness-with-prometheus/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">March 31, 2026</div> <div class="ts-meta-reading-time">5 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/from-custom-to-open-scalable-network-probing-and-http-3-readiness-with-prometheus/"><span>From Custom to Open: Scalable Network Probing and HTTP/3 Readiness with Prometheus</span></a></h2><span class="ts-mention">@Slack Engineering</span> </div>
</div>
</article>
</div>
</div>
</div>
<!-- Arc -->
<div class="c-arc concave-up">
<div class="c-arc__upper color-blue"></div>
<div class="c-arc__lower color-white"></div>
</div>
</div>
</main>
<footer class="ts-footer">
<div class="ts-container">
<div class="ts-footer__item">
<div class="ts-footer__col ts-col-left">
<div class="ts-footer__inner">
<div class="ts-logo">
<a class="ts-footer__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
</div>
<div class="ts-footer__col ts-col-canter">
<div class="ts-footer__inner">
<nav class="ts-footer__nav">
<div class="ts-footer__nav-item">
<ul class="ts-footer__nav-inner">
<li><a href="https://slack.engineering/articles/">Articles</a></li>
<!--<li><a href="https://slack.engineering/about/">About</a></li>-->
<li><a href="https://slack.com/careers/dept/software-engineering" target="_blank">Careers</a></li>
<li><a href="https://slack.dev/blog/" target="_blank">Slack Developer Blog</a></li>
<li><a href="https://slack.com/blog/" target="_blank">The Slack Blog</a></li>
</ul>
</div>
</nav>
</div>
</div>
<div class="ts-footer__col ts-col-right">
<div class="ts-footer__inner">
<span>
<a href="https://x.com/SlackEng" target="_blank" class="ts-share-buttons-twitter">
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M9.70487 6.85148L15.289 0.5H13.9657L9.11705 6.0149L5.24443 0.5H0.777832L6.63398 8.8395L0.777832 15.5H2.10116L7.22147 9.67608L11.3112 15.5H15.7778L9.70455 6.85148H9.70487ZM7.89239 8.91297L7.29905 8.08255L2.57797 1.47476H4.61052L8.42048 6.80746L9.01383 7.63788L13.9663 14.5696H11.9338L7.89239 8.91329V8.91297Z" fill="currentcolor"></path>
</svg>
</a>
<a href="https://www.youtube.com/@Slackhq" target="_blank" class="ts-share-buttons-youtube">
<svg width="21" height="14" viewBox="0 0 21 14" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M10.463 0C19.0721 0 20.1483 0 20.1483 7C20.1483 14 19.0721 14 10.463 14C1.85397 14 0.777832 14 0.777832 7C0.777832 0 1.85397 0 10.463 0ZM8.46143 2.7998C8.10631 2.7998 7.77271 3.0798 7.77271 3.4998V10.4998C7.77271 10.9521 8.11707 11.1998 8.46143 11.1998C8.60133 11.1998 8.7197 11.1567 8.82732 11.0921L14.3694 7.59211C14.5846 7.47365 14.6923 7.26904 14.6923 6.9998C14.6923 6.73057 14.5846 6.52596 14.3694 6.4075L8.82732 2.9075C8.70894 2.83211 8.59057 2.7998 8.46143 2.7998Z" fill="currentcolor"></path>
</svg>
</a>
</span>
</div>
</div>
</div>
<br/><br/>
<div class="ts-footer__item">
<p class="ts-footer__info">
<a href="https://slack.com/terms-of-service/user" target="_blank">Terms of Service</a>
<a href="https://slack.com/trust/privacy/privacy-policy" target="_blank">Privacy Information</a>
<a href="#" class="optanon-show-settings">Cookie Preferences</a>
<a href="https://www.salesforce.com/form/other/privacy-request/?_gl=1*1yvb56x*_gcl_au*OTY0OTg5MzcuMTcyNTkxMjE5Nw..*_ga*MTMzNjk3MzQ3MC4xNzE4MDQ0MzEy*_ga_QTJQME5M5D*MTcyNjI2OTEwNy40OS4xLjE3MjYyNjkyMzAuNS4wLjA." target="_blank">Your Privacy Choices</a>
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="30" height="14"><path d="M7.4 12.8h6.8l3.1-11.6H7.4C4.2 1.2 1.6 3.8 1.6 7s2.6 5.8 5.8 5.8z" style="fill-rule:evenodd;clip-rule:evenodd;fill:#fff"/><path d="M22.6 0H7.4c-3.9 0-7 3.1-7 7s3.1 7 7 7h15.2c3.9 0 7-3.1 7-7s-3.2-7-7-7zm-21 7c0-3.2 2.6-5.8 5.8-5.8h9.9l-3.1 11.6H7.4c-3.2 0-5.8-2.6-5.8-5.8z" style="fill-rule:evenodd;clip-rule:evenodd;fill:#06f"/><path d="M24.6 4c.2.2.2.6 0 .8L22.5 7l2.2 2.2c.2.2.2.6 0 .8-.2.2-.6.2-.8 0l-2.2-2.2-2.2 2.2c-.2.2-.6.2-.8 0-.2-.2-.2-.6 0-.8L20.8 7l-2.2-2.2c-.2-.2-.2-.6 0-.8.2-.2.6-.2.8 0l2.2 2.2L23.8 4c.2-.2.6-.2.8 0z" style="fill:#fff"/><path d="M12.7 4.1c.2.2.3.6.1.8L8.6 9.8c-.1.1-.2.2-.3.2-.2.1-.5.1-.7-.1L5.4 7.7c-.2-.2-.2-.6 0-.8.2-.2.6-.2.8 0L8 8.6l3.8-4.5c.2-.2.6-.2.9 0z" style="fill:#06f"/></svg>
<br/><br/>
&copy; 2026 Slack Technologies, LLC, a Salesforce company. All rights reserved. Various trademarks held by their respective owners.
</p>
</div>
</div>
</footer>
</div>
</div>
<a href="#top" data-scroll-to-top class="ts-scroll-to-top">
<svg class="ts-icon-up" width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M5 12H19M19 12L12 5M19 12L12 19" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
<span>scroll to top</span>
</a>
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/sites/7/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/tinyspeck/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script type="text/javascript" src="https://slack.engineering/_static/??-eJydj0sOwjAMRC9EaypVFBaIs6SJFRwSN4odUG9PI2DDCrGczxtp4JE7u7AiK+RYPbFALiTJKFmINL8UhLe9lQv2QXbwOxgN+67Uef2Hu1P6YMQ2VofSMkrGo8TFOHR9Iv6e1iumranEq2S0NzAiqAJSbMOFtL24pPMwHafDfjwNY3gCS2ZjcQ==" ></script><script id="jetpack-stats-js-before">
_stq = window._stq || [];
_stq.push([ "view", {"v":"ext","blog":"195077423","post":"16966","tz":"-7","srv":"slack.engineering","hp":"vip","j":"1:16.1.3"} ]);
_stq.push([ "clickTrackerInit", "195077423", "16966" ]);
//# sourceURL=jetpack-stats-js-before
</script>
<script data-wp-strategy="defer" defer fetchpriority="low" id="jetpack-stats-js" src="https://stats.wp.com/e-202637.js"></script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://slack.engineering/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://slack.engineering/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>