Files
nexus/sreweekly/articles/304/03-the-case-of-the-recursive-resolvers.html
2026-09-12 17:23:01 +08:00

674 lines
101 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<html lang="en-US">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, height=device-height, initial-scale=1.0, maximum-scale=1">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<link rel="profile" href="https://gmpg.org/xfn/11" />
<link rel="shortcut icon" href="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/icons/favicon.png" sizes="16x16 32x32 48x48" type="image/png">
<!-- Optanon Consent Notice start -->
<script src="https://d34u8crftukxnk.cloudfront.net/onetrust/slack.engineering/production/scripttemplates/otSDKStub.js" type="text/javascript" charset="UTF-8" data-domain-script="019872bf-9ff8-7d80-8158-f0c910bcdfad"></script>
<!-- Optanon Consent Notice end -->
<title>The Case of the Recursive Resolvers | Engineering at Slack</title>
<meta name="description" content="On September 30th 2021, Slack had an outage that impacted less than 1% of our online user base, and lasted for 24 hours. This outage was the result of our attempt to enable DNSSEC — an extension intended to secure the DNS protocol, required for FedRAMP Moderate — but which ultimately led to a series of&hellip;">
<meta property="og:url" content="https://slack.engineering/what-happened-during-slacks-dnssec-rollout/">
<meta property="og:type" content="website">
<meta property="og:title" content="The Case of the Recursive Resolvers">
<meta property="og:description" content="On September 30th 2021, Slack had an outage that impacted less than 1% of our online user base, and lasted for 24 hours. This outage was the result of our attempt to enable DNSSEC — an extension intended to secure the DNS protocol, required for FedRAMP Moderate — but which ultimately led to a series of&hellip;">
<meta property="og:image" content="https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg">
<meta property="og:image:width" content="6000">
<meta property="og:image:height" content="4000">
<meta name="twitter:card" content="summary_large_image">
<meta property="twitter:domain" content="slack.engineering">
<meta property="twitter:url" content="https://slack.engineering/what-happened-during-slacks-dnssec-rollout/">
<meta name="twitter:title" content="The Case of the Recursive Resolvers">
<meta name="twitter:description" content="On September 30th 2021, Slack had an outage that impacted less than 1% of our online user base, and lasted for 24 hours. This outage was the result of our attempt to enable DNSSEC — an extension intended to secure the DNS protocol, required for FedRAMP Moderate — but which ultimately led to a series of&hellip;">
<meta name="twitter:image" content="https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg">
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-590Q1VHB01"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-590Q1VHB01');
</script>
<!-- End Google Analytics -->
<title>The Case of the Recursive Resolvers &#8211; Engineering at Slack</title>
<meta name='robots' content='max-image-preview:large' />
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://slack.engineering/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fslack.engineering%2Fwhat-happened-during-slacks-dnssec-rollout%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://slack.engineering/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fslack.engineering%2Fwhat-happened-during-slacks-dnssec-rollout%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='all-css-12' href='https://slack.engineering/_static/??/wp-content/plugins/slack-config/style/shortcodes.css,/wp-content/themes/tinyspeck/assets/public/css/site.css?m=1783352904' type='text/css' media='all' />
<script type="text/javascript" src="https://slack.engineering/_static/??-eJyFi0sOgCAQQy+kjiRGdGE8i0HEQRiUT4y3FxL3rprXvsJ91sJRlBThNEkhBQhmEUdpN1SgM+/OR+FWGRodKsgPJGFS5rLqK0n/fNFYpF+ptqj8EuUnz3ZifOB9242s0y+jLDUm" ></script><link rel="https://api.w.org/" href="https://slack.engineering/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://slack.engineering/wp-json/wp/v2/posts/14285" /> <style>img#wpstats{display:none}</style>
<link rel="icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=32" sizes="32x32" />
<link rel="icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=192" sizes="192x192" />
<link rel="apple-touch-icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=180" />
<meta name="msapplication-TileImage" content="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=270" />
</head>
<body class="wp-singular post-template-default single single-post postid-14285 single-format-standard wp-embed-responsive wp-theme-tinyspeck">
<a class="ts-skip-link" href="#main">Skip to main content</a>
<div class="ts-site">
<div class="ts-site-inner">
<div class="ts-header-before"></div>
<header class="ts-header">
<div class="ts-container">
<div class="ts-header__inner ts-header__inner-desktop">
<div class="ts-header__col ts-col-left">
<div class="ts-logo">
<a class="ts-header__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
<div class="ts-header__col ts-col-center"></div>
<div class="ts-header__col ts-col-right">
<!--<a href="https://slack.engineering/articles/">Articles</a>
<a href="https://slack.engineering/about/">About</a>-->
<span class="ts-header__search-toggle" role="button" data-search-toggle="open">
<span class="ts-header__search-toggle-label">Search</span>
<svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</span>
</div>
</div>
<div class="ts-header__inner ts-header__inner-mobile">
<div class="ts-header__col ts-col-left"></div>
<div class="ts-header__col ts-col-center">
<div class="ts-logo">
<a class="ts-header__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
<div class="ts-header__col ts-col-right">
<span class="ts-header__search-toggle" role="button" data-search-toggle="open">
<span class="ts-header__search-toggle-label">Search</span>
<svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</span>
</div>
</div>
</div>
<div class="ts-search">
<div class="ts-container">
<div class="ts-search__close">
<div class="ts-search__close-inner" role="button" data-search-toggle="close">
<span class="ts-search__close-label">Close</span>
<svg class="ts-search__close-icon ts-icon ts-icon-x" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M18 6L6 18M6 6L18 18" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</div>
</div>
<div class="ts-search__wrapper">
<form role="search" method="get" class="ts-search__nav-form" action="https://slack.engineering/">
<div class="ts-search__group">
<input class="ts-search__input" type="search" required value="" name="s" placeholder="What are you looking for?">
<button class="ts-search__submit">Search</button>
</div>
</form>
</div>
</div>
</div>
</header>
<main id="main" class="ts-site-primary">
<div class="ts-site-content ts-sidebar-enabled ts-singular-thumbnail-enabled">
<div class="ts-container">
<div class="ts-main-content">
<div class="ts-content-area">
<div class="ts-entry__header ts-entry__header-large">
<div class="ts-entry__header-inner">
<div class="ts-entry__header-info">
<div class="ts-entry__post-meta-wrapper">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">November 29, 2021</div>
<div class="ts-meta-reading-time">18 min read</div> </div>
</div>
<h1 class="ts-jumbo"><span>The Case of the Recursive Resolvers</span></h1> <p class="ts-hero-paragraph">What Happened During Slack’s DNSSEC Rollout</p>
<div class="c-article__author__meta"><div class="c-article__author"><div class="c-article__author__avatar"><img src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/E12KS1G65-W0142PNSH1Q-29af50ce62dd-512.jpeg"/></div><div class="c-article__author__info"><span class="c-article__author__name">Rafael Elvira</span><span class="c-article__author__title">Staff Software Engineer, Infrastructure</span></div></div><div class="c-article__author"><div class="c-article__author__avatar"><img src="https://slack.engineering/wp-content/uploads/sites/7/2020/07/E12KS1G65-WJ5E9CNF7-d613b16ac25c-512.jpeg"/></div><div class="c-article__author__info"><span class="c-article__author__name">Laura Nolan</span><span class="c-article__author__title">Senior Staff Engineer</span></div></div></div> </div>
<figure class="ts-entry__post-media post-media filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg?w=1020" class="attachment-post-thumbnail size-post-thumbnail wp-post-image" alt="" decoding="async" fetchpriority="high" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg 5929w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg?resize=640,427 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg?resize=768,512 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg?resize=1280,853 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg?resize=1536,1024 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg?resize=2048,1365 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg?resize=380,253 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg?resize=800,533 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/torsten-dederichs-KrQJzrZiCak-unsplash.jpg?resize=1160,773 1160w" sizes="(max-width: 1020px) 100vw, 1020px" /> </figure>
</div>
</div>
<div class="ts-entry__wrap">
<div class="ts-entry__container">
<aside class="ts-widget-area ts-sidebar__area">
<div class="ts-sidebar__inner">
<!-- Search -->
<div class="widget widget_search">
<p class="ts-section-heading">Search</p>
<form role="search" method="get" class="ts-search__form" action="https://slack.engineering/">
<div class="ts-search__container">
<input required class="ts-search__input" type="search" value="" name="s" placeholder="">
<button class="ts-search__submit"><svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg></button>
</div>
</form>
</div>
<!-- Latest Posts -->
<div class="widget">
<p class="ts-section-heading">Latest Posts</p>
<div class="widget-body ts-widget-posts">
<ul>
<li class="ts-post-item">
<article class="post-17832 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-migration ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">July 14, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/">Shipyard: How We Built Slack’s Next-Generation EC2 Platform</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17854 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-automation-testing tag-developer-productivity tag-innovation ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">June 11, 2026</div> <div class="ts-meta-reading-time">12 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/agentic-testing-where-agents-fit-in-the-e2e-testing-stack/">Agentic Testing: Where Agents Fit in the E2E Testing Stack</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17812 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-backend tag-cloud-computing tag-collaboration tag-engineering tag-infrastructure tag-innovation tag-machine-learning tag-software-development ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">May 28, 2026</div> <div class="ts-meta-reading-time">17 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/">Slack AI: The Path to Multi-Cloud</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17774 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-airflow tag-aws tag-big-data tag-data-engineering tag-data-infrastructure tag-security ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">May 5, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/">From SSH to REST: A Security-Driven Modernization of Slack&#8217;s EMR Data Pipelines</a>
</h2>
</div>
</div>
</article>
</li>
<li class="ts-post-item">
<article class="post-17752 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-development tag-security tag-software-engineering ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta ">
<div class="ts-meta-date">April 13, 2026</div> <div class="ts-meta-reading-time">14 min read</div> </div>
<h2 class="ts-entry__title">
<a href="https://slack.engineering/managing-context-in-long-run-agentic-applications/">Managing context in long-run agentic applications</a>
</h2>
</div>
</div>
</article>
</li>
</ul>
</div>
</div>
<!-- Archives -->
<div class="widget">
<p class="ts-section-heading">Archives</p>
<ul class="wp-block-archives-list wp-block-archives">
<li><a href='https://slack.engineering/2026/'>2026</a>&nbsp;(7)</li>
<li><a href='https://slack.engineering/2025/'>2025</a>&nbsp;(9)</li>
<li><a href='https://slack.engineering/2024/'>2024</a>&nbsp;(19)</li>
<li><a href='https://slack.engineering/2023/'>2023</a>&nbsp;(16)</li>
<li><a href='https://slack.engineering/2022/'>2022</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2021/'>2021</a>&nbsp;(24)</li>
<li><a href='https://slack.engineering/2020/'>2020</a>&nbsp;(26)</li>
<li><a href='https://slack.engineering/2019/'>2019</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2018/'>2018</a>&nbsp;(11)</li>
<li><a href='https://slack.engineering/2017/'>2017</a>&nbsp;(21)</li>
<li><a href='https://slack.engineering/2016/'>2016</a>&nbsp;(19)</li>
</ul>
</div>
</div>
</aside>
<div class="ts-entry__content-wrap">
<div class="ts-single-post ts-wysiwyg">
<p>On September 30th 2021, Slack had an <a href="https://status.slack.com/2021-09-30">outage</a> that impacted less than 1% of our online user base, and lasted for 24 hours. This outage was the result of our attempt to enable DNSSEC — an extension intended to secure the DNS protocol, required for <a href="https://csrc.nist.gov/projects/risk-management/sp800-53-controls/release-search#!/control">FedRAMP Moderate</a> — but which ultimately led to a series of unfortunate events.</p>
<p>The internet relies very heavily on the Domain Name System (<a href="https://en.wikipedia.org/wiki/Domain_Name_System">DNS</a>) protocol. DNS is like a phone book for the entire internet. Web sites are accessed through <a href="https://en.wikipedia.org/wiki/Domain_name">domain names</a>, but web browsers interact using <a href="https://en.wikipedia.org/wiki/IP_address">IP addresses</a>. DNS translates domain names to IP addresses, so that browsers can load the sites you need. Refer to <a href="https://www.cloudflare.com/en-us/learning/dns/what-is-dns/">‘What is DNS?’ by Cloudflare</a> to read more about how DNS works and all the necessary steps to do a domain name lookup.</p>
<p><img decoding="async" class="alignnone wp-image-14286" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-dns-path-example.png?w=640" alt="" width="667" height="134" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-dns-path-example.png 3263w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-dns-path-example.png?resize=640,129 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-dns-path-example.png?resize=768,154 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-dns-path-example.png?resize=1280,257 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-dns-path-example.png?resize=1536,309 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-dns-path-example.png?resize=2048,412 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-dns-path-example.png?resize=380,76 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-dns-path-example.png?resize=800,161 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-dns-path-example.png?resize=1160,233 1160w" sizes="(max-width: 667px) 100vw, 667px" /></p>
<p>DNS as a protocol is insecure by default, and anyone in transit between the client and the <a href="https://en.wikipedia.org/wiki/Name_server#Authoritative_name_server">authoritative DNS name server</a> for a given domain name can tamper with the response, directing the client elsewhere. DNS has a security extension commonly referred to as <a href="https://en.wikipedia.org/wiki/Domain_Name_System_Security_Extensions">DNSSEC</a>, which prevents tampering with responses between the authoritative DNS server of the domain name (i.e. slack.com.) and the client&#8217;s DNS recursive resolver of choice. DNSSEC will not protect the last mile of DNS — which is the communication between the client and their DNS recursor — from a <a href="https://en.wikipedia.org/wiki/Man-in-the-middle_attack">MiTM attack</a>. While we are aware of the <a href="https://sockpuppet.org/blog/2015/01/15/against-dnssec/">debate around the utility of DNSSEC</a> among the DNS community, we are still committed to securing Slack for our customers.</p>
<p><img decoding="async" width="2698" height="1794" class="alignnone size-medium wp-image-14287" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-DNSSEC-secure-path.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-DNSSEC-secure-path.png 2698w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-DNSSEC-secure-path.png?resize=640,426 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-DNSSEC-secure-path.png?resize=768,511 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-DNSSEC-secure-path.png?resize=1280,851 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-DNSSEC-secure-path.png?resize=1536,1021 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-DNSSEC-secure-path.png?resize=2048,1362 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-DNSSEC-secure-path.png?resize=380,253 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-DNSSEC-secure-path.png?resize=800,532 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-DNSSEC-secure-path.png?resize=1160,771 1160w" sizes="(max-width: 2698px) 100vw, 2698px" /></p>
<p>At Slack, we use <a href="https://aws.amazon.com/route53/">Amazon Route 53</a> as our authoritative DNS server for all our public domains, delegating some subzones to <a href="https://ns1.com/">NS1</a>. We do this to make use of NS1’s advanced traffic management features. All these zones are operated by the Traffic Engineering team at Slack.</p>
<p>For the last four months, the team has been working on enabling DNSSEC signing on all <a href="https://en.wikipedia.org/wiki/Domain_name">domains</a> used by Slack. At the time of the incident, all critical Slack domains, other than slack.com, already had DNSSEC enabled for at least one month.</p>
<p>By design, DNSSEC can only be enabled on a per-domain basis, making it impossible to enable DNSSEC per subdomain (i.e. example.slack.com) or subzone (i.e. wss-backup.slack.com, which is delegated from the slack.com Route53 zone to NS1). This design choice significantly increases the risk of DNSSEC rollouts.</p>
<p>Accordingly, we were careful about testing that our domains were still resolving correctly. For each domain that we enabled DNSSEC on we did the following validation:</p>
<ul>
<li>We created external monitoring and alerting for DNS and DNSSEC resolution from multiple locations around the globe using many resolvers (including Cloudflare, Comodo, Google, OpenDNS, Quad9, and Verisign).</li>
<li>We created multiple <a href="https://dnsviz.net">dnsviz.net</a> and <a href="https://dnssec-debugger.verisignlabs.com/">Verisign DNSSEC Debugger</a> tests to make sure that our zones were set up correctly, and that the chain of trust between authoritative and delegated zones was successful.</li>
<li>We created multiple <a href="https://dnschecker.org">dnschecker.org</a> tests to ensure resolution against slack.com and it’s sub-delegated records resolved as expected after applying the zone changes.</li>
<li>We created several hand-crafted <a href="https://en.wikipedia.org/wiki/Dig_(command)">dig</a> tests using multiple resolvers.</li>
</ul>
<p>Prior to enabling DNSSEC for slack.com, we rolled out DNSSEC to all other Slack public domains successfully, with no internal or customer impact. The incident on September 30th 2021 was the third unsuccessful attempt to enable DNSSEC on slack.com.</p>
<p><img loading="lazy" decoding="async" width="2968" height="2588" class="alignnone size-large wp-image-14291" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-Copy-of-dnssec-rollout-timeline-1.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-Copy-of-dnssec-rollout-timeline-1.png 2968w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-Copy-of-dnssec-rollout-timeline-1.png?resize=640,558 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-Copy-of-dnssec-rollout-timeline-1.png?resize=768,670 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-Copy-of-dnssec-rollout-timeline-1.png?resize=1280,1116 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-Copy-of-dnssec-rollout-timeline-1.png?resize=1536,1339 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-Copy-of-dnssec-rollout-timeline-1.png?resize=2048,1786 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-Copy-of-dnssec-rollout-timeline-1.png?resize=380,331 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-Copy-of-dnssec-rollout-timeline-1.png?resize=800,698 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-Copy-of-dnssec-rollout-timeline-1.png?resize=1160,1011 1160w" sizes="auto, (max-width: 2968px) 100vw, 2968px" /></p>
<h2>Unsuccessful attempt one 😟</h2>
<p>On September 7th, 2021, we enabled DNSSEC signing on slack.com. However, prior to publishing the DS record at our <a href="https://en.wikipedia.org/wiki/Domain_name_registrar">DNS registrar</a>, a large <a href="https://en.wikipedia.org/wiki/Internet_service_provider">ISP</a> in the US had an outage and many of our customers reported being unable to access Slack as a result. Early in that incident — before we were aware of the ISP issue — the <a href="https://slack.com">slack.com</a> zone changes were rolled back as a precaution.</p>
<h2>Unsuccessful attempt two 😢</h2>
<p>On September 8th, 2021, we enabled DNSSEC signing on slack.com following the same procedure as on our previous attempt. Prior to publishing the DS record at the DNS registrar, our Customer Experience team noticed a small number of users reporting DNS resolution problems. Most of the affected users were accessing Slack through a VPN provider.</p>
<p>It turned out that some resolvers become more strict when DNSSEC signing is enabled at the authoritative name servers, <i>even while signing was not enabled at the root name servers (i.e. before DS records were published to COM nameservers).</i> This strict DNS spec enforcement will reject a <a href="https://en.wikipedia.org/wiki/CNAME_record">CNAME record</a> at the apex of a zone (as per <a href="https://www.rfc-editor.org/rfc/rfc1912#section-2.4">RFC-2181</a>), including the APEX of a sub-delegated subdomain. This was the reason that customers using VPN providers were disproportionately affected — many VPN providers use resolvers with this behavior.</p>
<p>We stopped signing the slack.com zone in Route53 and NS1. Shortly afterwards, this fixed the DNS resolution issues for users that had reported problems. After this attempt, our sub-delegated zones were updated to <a href="https://help.ns1.com/hc/en-us/articles/360017511293-What-is-the-difference-between-CNAME-and-ALIAS-records-">ALIAS records</a>.</p>
<h2>Unsuccessful attempt three 😭</h2>
<p>On September 30th 2021 and after careful planning, the Traffic Engineering team made their third attempt to enable DNSSEC signing on slack.com. We started by enabling DNSSEC signing on all NS1 <a href="https://slack.com">slack.com</a> sub-delegated zones and the Route53 authoritative zone for <a href="https://slack.com">slack.com</a>, which also provisioned the necessary DS records from the delegated zones to maintain chain of trust between Authoritative and delegated zones.</p>
<p>Immediately after, we followed all the validation steps listed above to ensure that slack.com was still resolving correctly.</p>
<p><img loading="lazy" decoding="async" width="3075" height="2436" class="alignnone size-large wp-image-14292" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-1.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-1.png 3075w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-1.png?resize=640,507 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-1.png?resize=768,608 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-1.png?resize=1280,1014 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-1.png?resize=1536,1217 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-1.png?resize=2048,1622 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-1.png?resize=380,301 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-1.png?resize=800,634 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-1.png?resize=1160,919 1160w" sizes="auto, (max-width: 3075px) 100vw, 3075px" /><i>State of slack.com DNS when DNSSEC signing was enabled at slack.com zone, but not yet published DS record at .com zone.</i></p>
<p>After a three hour ‘soak time’ and successful validations, we were confident to publish the slack.com <a href="https://www.cloudflare.com/en-gb/learning/dns/dns-records/dnskey-ds-records/">DS record</a> at the DNS registrar, which instructed DNSSEC validating resolvers to start validating DNS requests for slack.com (unless the <a href="https://datatracker.ietf.org/doc/html/rfc4035#section-3.2.2">cd flag</a> is specified).</p>
<p>After publishing the DS record at the DNS registrar, we ran the same validation steps again and confirmed everything was working as intended. Our DNS resolution monitoring that enforced DNSSEC checking corroborated a successful rollout.</p>
<p><img loading="lazy" decoding="async" width="3638" height="2481" class="alignnone size-large wp-image-14293" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-2.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-2.png 3638w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-2.png?resize=640,436 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-2.png?resize=768,524 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-2.png?resize=1280,873 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-2.png?resize=1536,1048 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-2.png?resize=2048,1397 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-2.png?resize=380,259 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-2.png?resize=800,546 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-2.png?resize=1160,791 1160w" sizes="auto, (max-width: 3638px) 100vw, 3638px" /><i>State of slack.com DNS chain after DNSSEC was fully enabled.</i></p>
<p>Our DNSSEC probes went green after publishing the DS record at the DNS registrar as well. Success!<img loading="lazy" decoding="async" width="1118" height="1042" class="alignnone size-large wp-image-14294" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.32.10.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.32.10.png 1118w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.32.10.png?resize=640,596 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.32.10.png?resize=768,716 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.32.10.png?resize=380,354 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.32.10.png?resize=800,746 800w" sizes="auto, (max-width: 1118px) 100vw, 1118px" /></p>
<p>Soon afterwards, Customer Experience started getting reports from users seeing “ERR_NAME_NOT_RESOLVED” errors in Slack clients. Slack applications are based on Chromium and integrate a way to export a <a href="https://www.chromium.org/developers/design-documents/network-stack/netlog">NetLog</a> capture, which is incredibly helpful when debugging network problems and analyzing performance. NetLogs are extensively used by Customer Experience and Engineering at Slack.</p>
<p>‘ERR_NAME_NOT_RESOLVED’ type errors can have multiple causes. In this specific case, it was caused by <a href="https://datatracker.ietf.org/doc/html/rfc2308#section-2.2">NODATA</a> DNS responses from some DNS recursive resolvers. Our engineering teams started to investigate.</p>
<p>During our initial investigation we saw most reports were from a private (corporate) DNS recursive resolver and Google’s Public Resolver (8.8.8.8). This was surprising, as impact to Google DNS <i>should</i> mean a much larger impact than our monitoring and customer reports indicated.</p>
<p>At this point we decided to pull the DS record from the DNS registrar, effectively removing the slack.com DS record at the ‘.com.’ zone to ‘stop the bleeding’, and later focus on the ‘NODATA’ DNS responses reported by customers.<img loading="lazy" decoding="async" width="3068" height="2436" class="alignnone size-large wp-image-14295" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4.png 3068w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4.png?resize=640,508 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4.png?resize=768,610 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4.png?resize=1280,1016 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4.png?resize=1536,1220 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4.png?resize=2048,1626 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4.png?resize=380,302 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4.png?resize=800,635 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4.png?resize=1160,921 1160w" sizes="auto, (max-width: 3068px) 100vw, 3068px" /><i>State of Slack.com DNS after removing the slack.com DS record from the .com zone</i></p>
<p>After nearly an hour, there were no signs of improvement and error rates remained stable, trending neither upwards nor downwards.</p>
<p>Our Traffic team was confident that reverting the DS record published in MarkMonitor was sufficient to eventually resolve DNS resolution problems. As things were not getting any better, and given the severity of the incident, we decided to rollback DNSSEC signing in our slack.com authoritative and delegated zones, wanting to recover our DNS configuration to the last previous healthy state, allowing us to completely rule out DNSSEC as a problem.</p>
<p><img loading="lazy" decoding="async" width="3068" height="2436" class="alignnone size-large wp-image-14296" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-5.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-5.png 3068w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-5.png?resize=640,508 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-5.png?resize=768,610 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-5.png?resize=1280,1016 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-5.png?resize=1536,1220 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-5.png?resize=2048,1626 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-5.png?resize=380,302 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-5.png?resize=800,635 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-5.png?resize=1160,921 1160w" sizes="auto, (max-width: 3068px) 100vw, 3068px" /><i>State of slack.com DNS after un-signing DNSSEC at slack.com zone</i></p>
<p>As soon as the rollback was pushed out things got much worse; our Traffic team was paged due to DNS resolution issues for slack.com from multiple resolvers across the world.</p>
<p><img loading="lazy" decoding="async" width="2028" height="1628" class="alignnone size-large wp-image-14298" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.45.22.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.45.22.png 2028w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.45.22.png?resize=640,514 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.45.22.png?resize=768,617 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.45.22.png?resize=1280,1028 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.45.22.png?resize=1536,1233 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.45.22.png?resize=380,305 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.45.22.png?resize=800,642 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.45.22.png?resize=1160,931 1160w" sizes="auto, (max-width: 2028px) 100vw, 2028px" /><i>Example DNS resolution alert for 9.9.9.9 DNS resolver received by our Traffic team after the rollback.</i></p>
<p>Based on expert advice, our understanding at the time was that DS records at the .com zone were never cached, so pulling it from the registrar would cause resolvers to immediately stop performing DNSSEC validation. This assumption turned out to be untrue. The ‘.com.’ zone will ask resolvers to cache the slack.com DS record for 24h by default (non modifiable by zone owners). Luckily, some resolvers cap that TTL with a lower value (i.e. Google’s 8.8.8.8 has a 6h TTL on DS records).</p>
<p><img loading="lazy" decoding="async" width="2888" height="2325" class="alignnone size-large wp-image-14299" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS-expectation.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS-expectation.png 2888w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS-expectation.png?resize=640,515 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS-expectation.png?resize=768,618 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS-expectation.png?resize=1280,1030 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS-expectation.png?resize=1536,1237 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS-expectation.png?resize=2048,1649 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS-expectation.png?resize=380,306 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS-expectation.png?resize=800,644 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS-expectation.png?resize=1160,934 1160w" sizes="auto, (max-width: 2888px) 100vw, 2888px" /><i>How we thought recursive resolvers would behave after disabling DNSSEC signing in slack.com zone</i></p>
<p>The team knew removing the DNSKEY from the authoritative zone is a high-risk operation unless the DS record is not published anymore. However, the assumption of no cached DS records at the ‘.com.’ zone made us believe we were in the same state as in the previous two rollbacks, and that the result would be the same. However, unlike with our previous attempt, this time resolvers had cached DS records with up to 24 TTL remaining.</p>
<p><img loading="lazy" decoding="async" width="2888" height="2325" class="alignnone size-large wp-image-14300" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS.png 2888w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS.png?resize=640,515 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS.png?resize=768,618 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS.png?resize=1280,1030 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS.png?resize=1536,1237 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS.png?resize=2048,1649 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS.png?resize=380,306 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS.png?resize=800,644 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-slackcom-dnssec-step-4TTLS.png?resize=1160,934 1160w" sizes="auto, (max-width: 2888px) 100vw, 2888px" /><i>How recursive resolvers actually behaved: they had cached the slack.com DS record for 24h. DNSSEC validations started to fail after disabling signing on slack.com zone.</i></p>
<p>All our DNS configuration is managed in <a href="https://www.terraform.io/">Terraform</a>. Terraform is great for managing infrastructure as code, but in this case it made us miss a <b>critical</b> warning when trying to disable DNSSEC signing in Route53:</p>
<p><img loading="lazy" decoding="async" width="1682" height="1484" class="alignnone size-large wp-image-14301" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.54.01.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.54.01.png 1682w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.54.01.png?resize=640,565 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.54.01.png?resize=768,678 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.54.01.png?resize=1280,1129 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.54.01.png?resize=1536,1355 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.54.01.png?resize=380,335 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.54.01.png?resize=800,706 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.54.01.png?resize=1160,1023 1160w" sizes="auto, (max-width: 1682px) 100vw, 1682px" /><i>Disable DNSSEC warning window in Route53 Console.</i></p>
<p>Once the rollback was complete, all DNSSEC validating resolvers that had recently cached the DS record for slack.com started returning a <a href="https://bluecatnetworks.com/blog/the-top-four-dns-response-codes-and-what-they-mean/">SERVFAIL</a> on all slack.com lookups. We were immediately alerted by our external monitoring on DNS/DNSSEC resolution probes.</p>
<p>Our team quickly started contacting major ISPs and operators that run public DNS resolvers, with the request to flush all cached records for slack.com. Additionally, we flushed caches for resolvers that offer a public site (i.e. <a href="https://developers.google.com/speed/public-dns/cache">Google’s 8.8.8.8</a> and <a href="https://1.1.1.1/purge-cache/">Cloudflare’s 1.1.1.1</a>).</p>
<p>As resolvers flushed their caches, we slowly saw signs of improvement on our internal monitoring. Slack clients send telemetry to slackb.com and thanks to this isolation, we were still able to receive client metrics that helped us quantify the impact of this outage. Ironically, slackb.com had DNSSEC enabled successfully prior to this incident.</p>
<p>Throughout the incident, we considered re-signing DNSSEC on the slack.com zone (without publishing the DS at the DNS registrar). Restoring the DNSKEY would slightly improve the situation for some users, but it would not have solved the issue with all users that were initially getting ‘NODATA’ on their responses after DNSSEC signing was enabled. With hindsight this might have been better, but at this point we still did not understand the root cause of the original ‘NODATA’ responses.</p>
<p>For us to restore signing, we would have had to recover some keys which had been deleted when we rolled back the changes to the slack.com zone. In Route53 the <a href="https://www.cloudflare.com/en-gb/dns/dnssec/how-dnssec-works/">DNSSEC KSK (Key Signing Key)</a> is managed via an <a href="https://aws.amazon.com/kms/">AWS KMS key</a>. The KMS key was recoverable as per our configuration and the KSK was going to be the same, but we were uncertain if the ZSK (Zone Signing Key) would have been the same or different, as the ZSK is fully managed by AWS and we have no visibility into it. After the incident, Amazon’s Route53 team confirmed that the ZSK would have been different. When re-enabling signing on a zone, the ZSK will be generated with a new key pair. The Route53 team intends to design a mechanism for customers to quickly recover from this situation in the future.</p>
<p>At this point, some hours into the incident, we were in a situation where Slack was resolving correctly for the vast majority of customers, but a long tail of users of smaller DNS resolvers — which had not flushed the slack.com records — were still impacted.</p>
<p>We decided not to restore signing due to a lack of confidence. At the time, our understanding was that the DS record TTL might have been set by the slack.com zone in Route53, so a failed attempt to re-enable signing on the zone would ‘reset the clock’ on the DS record TTL, so we would have to request flushing of the slack.com DNS records for a second time and prolong the impact of the incident.</p>
<p>We know now signing a zone has no impact on the DS record TTL as it comes from the .com zone directly. Re-enabling signing would have had no impact as the ZSK would be different, so all DNSSEC validating resolvers would have continued failing to validate. So this action would have neither made things better nor made them worse.</p>
<p>As public resolver caches were flushed and as the 24h TTL on the DS record at the ‘.com’ zone expired, the error rates went back to normal.</p>
<h2>Solving the mystery 🕵️</h2>
<p>Once impact to Slack customers was mitigated, we aimed to determine the root cause of this outage. Prior to attempting DNSSEC on slack.com, our team had successfully enabled DNSSEC on all other Slack public domains. So how was slack.com different? Why had we not seen issues with any of the previous domains? Why did it only impact <i>some </i>corporate internal DNS resolvers and <i>some </i>of Google’s Public DNS lookups, but not all?</p>
<p>To understand what had happened, we replicated the rollout steps using a test zone setup identical to slack.com and used a resolver that we knew had problems during the rollout. Surprisingly, all our attempts to reproduce the behavior were unsuccessful.</p>
<p>At this point we went back to the NetLogs that we had received from customers during the outage.</p>
<table>
<tbody>
<tr>
<td>Side Note: How do we get access to NetLogs from our customers? These were voluntarily provided by some customers after they opened support cases during this incident, so a <b>huge thank you</b> to those of you that did that! It really helps us out when debugging incidents like this one.</td>
</tr>
</tbody>
</table>
<p>This is where we found an extremely interesting clue: clients were able to resolve slack.com successfully but failed to resolve app.slack.com. Why?</p>
<p>The following screenshot shows an attempt to resolve the slack.com domain successfully at “At 12:32:59.120”: <img loading="lazy" decoding="async" width="1252" height="606" class="alignnone size-large wp-image-14302" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.55.34.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.55.34.png 1252w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.55.34.png?resize=640,310 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.55.34.png?resize=768,372 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.55.34.png?resize=380,184 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.55.34.png?resize=800,387 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.55.34.png?resize=1160,561 1160w" sizes="auto, (max-width: 1252px) 100vw, 1252px" /></p>
<p>One microsecond later, app.slack.com fails to resolve with a ‘ERR_NAME_NOT_RESOLVED’ error:<img loading="lazy" decoding="async" width="1258" height="590" class="alignnone size-large wp-image-14303" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.56.07.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.56.07.png 1258w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.56.07.png?resize=640,300 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.56.07.png?resize=768,360 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.56.07.png?resize=380,178 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.56.07.png?resize=800,375 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/Screenshot-2021-11-05-at-11.56.07.png?resize=1160,544 1160w" sizes="auto, (max-width: 1258px) 100vw, 1258px" /></p>
<p>This indicated there was likely a problem with the ‘*.slack.com’ wildcard record since we didn’t have a wildcard record in any of the other domains where we had rolled out DNSSEC on. Yes, it was an oversight that we did not test a domain with a wildcard record before attempting slack.com — learn from our mistakes!</p>
<p>We collected evidence and reached out to the Amazon Route 53 team, who jumped on a call with us to figure it out. After walking them through the evidence we had, they quickly correlated this behavior to an issue with <a href="https://en.wikipedia.org/wiki/Domain_Name_System_Security_Extensions#Authenticating_NXDOMAIN_responses_and_NSEC">NSEC responses</a> that Route 53 generated on wildcard records. Aha! 💡</p>
<p>When a request is made to a DNSSEC-enabled domain for a wildcard record which type does not exist, the answer is a signed NSEC record confirming that while the domain name exists, the requested type does not. This signed NSEC response can include some additional information (<a href="https://datatracker.ietf.org/doc/html/rfc7129#section-5.3">specifically designed to keep wildcard records from breaking, among other things</a>), but Route53 was not returning this additional information.</p>
<p>What this means is that a request for a record at the same level, say an <a href="https://en.wikipedia.org/wiki/List_of_DNS_record_types">AAAA lookup</a> for <a href="http://app.slack.com/">app.slack.com</a> would correctly return a signed NSEC saying “there is no record for that”. This is correct since we don’t publish any AAAA records. The expected client behaviour is to fall back to requesting an A record. BUT — since Route53 isn’t returning that extra information — some public resolvers (including Google Public DNS, 8.8.8.8) take that response to mean that <b>nothing</b> exists at that level so there’s no need to query for an A record.</p>
<p>This NSEC response is cached. This effectively meant that impacted DNS resolvers were unintentionally being “cache poisoned” (by clients innocently checking for the existence of AAAA records) when DNSSEC was enabled in a Route53 zone with a wildcard record. Luckily, this cache poisoning would only affect the regional cache. This meant that one regional cache backing a larger impacted resolver could be temporarily poisoned, and anyone else hitting that same cache would see the failure, but those hitting other regional caches on the same resolver would not.</p>
<p><img loading="lazy" decoding="async" width="3441" height="2554" class="alignnone size-large wp-image-14304" src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-cache-poisoning-example.png?w=688" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-cache-poisoning-example.png 3441w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-cache-poisoning-example.png?resize=640,475 640w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-cache-poisoning-example.png?resize=768,570 768w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-cache-poisoning-example.png?resize=1280,950 1280w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-cache-poisoning-example.png?resize=1536,1140 1536w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-cache-poisoning-example.png?resize=2048,1520 2048w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-cache-poisoning-example.png?resize=380,282 380w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-cache-poisoning-example.png?resize=800,594 800w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/DNSSEC-blog-post-diagrams-cache-poisoning-example.png?resize=1160,861 1160w" sizes="auto, (max-width: 3441px) 100vw, 3441px" /><i>Recursive resolvers caches are poisoned by incorrect NSEC response from Route53.</i></p>
<p>Interestingly, at this point we were able to reproduce this issue with our DNSSEC-enabled test zone against Google Public DNS (8.8.8.8) but <b>not</b> against Cloudflare Public DNS (1.1.1.1). A second mystery was whether or not one of these major public resolvers was out of spec. We investigated this and decided it is protocol compliant to reuse a cached NSEC record to decide a subsequent response. In fact, that&#8217;s what Aggressive NSEC (<a href="https://research.google/pubs/pub46737/">RFC 8198</a>) does, an optional spec, which Google implements. Some implementations only infer non-existence of &#8220;names&#8221; within the NSEC span. Some also do RR &#8220;type&#8221; inference from the NSEC-type bitmap. It seems that Google does both and perhaps Cloudflare does neither (or only the first). In the end we concluded that both options were “acceptable” and the only real way to avoid being in this ambiguous situation is for your Authoritative Nameserver to correctly return existing record types in the NSEC type bitmap (which was exactly what the AWS Route53 team was fixing).</p>
<p><i>Lookup of existing A wildcard record ‘*.slackexperts.com‘:</i></p>
<pre class="c-mrkdwn__pre" data-stringify-type="pre">dig A qqq.slackexperts.com @8.8.8.8 +dnssec
; &lt;&lt;&gt;&gt; DiG 9.10.6 &lt;&lt;&gt;&gt; A qqq.slackexperts.com @8.8.8.8 +dnssec
;; global options: +cmd
;; Got answer:
;; -&gt;&gt;HEADER&lt;&lt;- opcode: QUERY, status: NOERROR, id: 48704
;; flags: qr rd ra ad; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 1
;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags: do; udp: 512
;; QUESTION SECTION:
;qqq.slackexperts.com. IN A
;; ANSWER SECTION:
qqq.slackexperts.com. 300 IN A 54.211.89.16
qqq.slackexperts.com. 300 IN RRSIG A 13 3 300 20211105130720 20211105110220 28453 slackexperts.com. zksghNF9JTx4mRXjkoEGP6C5zCPb5JjX2MzeihENzx3JIjnZLZyokvdd /dnxcA5Qjl3sc3eC0bkryoGATb4ghw==
;; Query time: 66 msec
;; SERVER: 8.8.8.8#53(8.8.8.8)
;; WHEN: Wed Oct 13 15:40:32 EDT 2021
;; MSG SIZE rcvd: 177</pre>
<p><i>Lookup of non existent AAAA wildcard record ‘*.slackexperts.com’ &#8211; no A specified in NSEC response:</i></p>
<pre class="c-mrkdwn__pre" data-stringify-type="pre">dig AAAA qqq.slackexperts.com @8.8.8.8 +dnssec
; &lt;&lt;&gt;&gt; DiG 9.10.6 &lt;&lt;&gt;&gt; AAAA qqq.slackexperts.com @8.8.8.8 +dnssec
;; global options: +cmd
;; Got answer:
;; -&gt;&gt;HEADER&lt;&lt;- opcode: QUERY, status: NOERROR, id: 63101
;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 4, ADDITIONAL: 1
;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags: do; udp: 512
;; QUESTION SECTION:
;qqq.slackexperts.com. IN AAAA
;; AUTHORITY SECTION:
slackexperts.com. 2370 IN SOA ns-869.awsdns-44.net. awsdns-hostmaster.amazon.com. 1 7200 900 1209600 86400
slackexperts.com. 2370 IN RRSIG SOA 13 2 900 20211013205618 20211013184118 30624 slackexperts.com. iBn8JmsuVq+F5zD+cm6C1Ypr25TtpKAIduCUZ80+sbuCbMQh0beC7RFc 2MctSnC9tf1RuCQHUxa/sEVVyHw6xQ==
qqq.slackexperts.com. 2370 IN NSEC \000.qqq.slackexperts.com. RRSIG NSEC
qqq.slackexperts.com. 2370 IN RRSIG NSEC 13 3 86400 20211014204118 20211013184118 30624 slackexperts.com. yB7d/Fl15M0z6zEHuE4XDvdxBT91DPiV38vhvI/42piqA1YI9ibb178E X0L82AaIaxcvSXKsdrWD8bAxJGVVyw==
;; Query time: 40 msec
;; SERVER: 8.8.8.8#53(8.8.8.8)
;; WHEN: Wed Oct 13 15:41:47 EDT 2021
;; MSG SIZE rcvd: 398</pre>
<p><i>Lookup of existing A wildcard record after cache is poisoned (no ANSWER SECTION &#8211; It’s empty!):</i></p>
<pre class="c-mrkdwn__pre" data-stringify-type="pre">dig A qqq.slackexperts.com @8.8.8.8 +dnssec
; &lt;&lt;&gt;&gt; DiG 9.10.6 &lt;&lt;&gt;&gt; A qqq.slackexperts.com @8.8.8.8 +dnssec
;; global options: +cmd
;; Got answer:
;; -&gt;&gt;HEADER&lt;&lt;- opcode: QUERY, status: NOERROR, id: 17738
;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 4, ADDITIONAL: 1
;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags: do; udp: 512
;; QUESTION SECTION:
;qqq.slackexperts.com. IN A
;; AUTHORITY SECTION:
slackexperts.com. 2382 IN SOA ns-869.awsdns-44.net. awsdns-hostmaster.amazon.com. 1 7200 900 1209600 86400
slackexperts.com. 2382 IN RRSIG SOA 13 2 900 20211013205618 20211013184118 30624 slackexperts.com. iBn8JmsuVq+F5zD+cm6C1Ypr25TtpKAIduCUZ80+sbuCbMQh0beC7RFc 2MctSnC9tf1RuCQHUxa/sEVVyHw6xQ==
qqq.slackexperts.com. 2382 IN NSEC \000.qqq.slackexperts.com. RRSIG NSEC
qqq.slackexperts.com. 2382 IN RRSIG NSEC 13 3 86400 20211014204118 20211013184118 30624 slackexperts.com. yB7d/Fl15M0z6zEHuE4XDvdxBT91DPiV38vhvI/42piqA1YI9ibb178E X0L82AaIaxcvSXKsdrWD8bAxJGVVyw==
;; Query time: 10 msec
;; SERVER: 8.8.8.8#53(8.8.8.8)
;; WHEN: Wed Oct 13 15:42:12 EDT 2021
;; MSG SIZE rcvd: 398</pre>
<p>Ta-da! 🎉</p>
<p>Our mystery was definitively solved. At the time of the writing of this blog post, the Route53 team has rolled out a fix to the NSEC-type bug described above.</p>
<p>We are very sorry for the impact this outage had on everyone who uses Slack and hope that this write-up will shed some light on some of the corner cases with DNSSEC (circa 2021). Maybe, just maybe, this will prevent someone else from having a similar issue.</p>
<h2>Acknowledgements 🙏</h2>
<p>We’d like to thank the Amazon Route 53 team for confirming the issue and for their rapid fix, Salesforce DNS team for their kindness, support, and DNSSEC expertise, and all of the DNS community for their invaluable help in rapidly flushing caches to bring Slack back to everyone affected by this problem. Last but not least, a huge thank you to our Customer Experience team, who are the ones on the front lines every day, connecting the dots between our customers and our engineering teams — we could not have done it without you! 🌟</p>
<p>1 <a href="https://csrc.nist.gov/projects/risk-management/sp800-53-controls/release-search#!/control">FedRAMP Moderate NIST 800-53 controls</a> (<a href="https://csrc.nist.gov/projects/risk-management/sp800-53-controls/release-search#!/control?version=5.1&amp;number=SC-20">SC-20</a>, <a href="https://csrc.nist.gov/projects/risk-management/sp800-53-controls/release-search#!/control?version=5.1&amp;number=SC-21">SC-21</a>)</p>
</div>
</div>
</div>
</div>
</div>
<aside class="ts-metabar__area">
<div class="ts-metabar__inner">
<div class="ts-metabar__item">
<div class="ts-share-buttons-wrap">
<div class="ts-share-buttons-items">
<div class="ts-share-buttons-item ts-share-buttons-twitter">
<a href="https://x.com/intent/post?url=https%3A%2F%2Fslack.engineering%2Fwhat-happened-during-slacks-dnssec-rollout%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-twitter" width="21" viewBox="0 0 72 65" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M22.6891 0H0L26.9014 35.3477L1.72089 65H13.355L32.3999 42.5731L49.3109 64.794H72L44.317 28.4191L44.366 28.4818L68.2015 0.412927H56.5674L38.8667 21.2573L22.6891 0ZM12.524 6.19052H19.5874L59.476 58.603H52.4126L12.524 6.19052Z" fill="currentColor"/></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-linkedin">
<a href="https://www.linkedin.com/shareArticle?mini=true&#038;url=https%3A%2F%2Fslack.engineering%2Fwhat-happened-during-slacks-dnssec-rollout%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-linkedin" width="21" height="21" viewBox="0 0 70 70" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M15.8066 23.0188H1.17969V69.663H15.8066V23.0188Z" fill="currentColor"/><path d="M55.3732 22.0077C54.834 21.9403 54.2611 21.9066 53.6881 21.8729C45.4984 21.5359 40.8812 26.389 39.2635 28.4786C38.8254 29.0515 38.6231 29.3886 38.6231 29.3886V23.1536H24.6366V69.7978H38.6231H39.2635C39.2635 65.0458 39.2635 60.3274 39.2635 55.5754C39.2635 53.014 39.2635 50.4526 39.2635 47.8912C39.2635 44.7232 39.0276 41.3529 40.6116 38.4545C41.9597 36.0279 44.3862 34.8147 47.1161 34.8147C55.2047 34.8147 55.3732 42.1281 55.3732 42.8021C55.3732 42.8358 55.3732 42.8695 55.3732 42.8695V70H70.0001V39.5667C70.0001 29.1526 64.7088 23.0188 55.3732 22.0077Z" fill="currentColor"/><path d="M8.49301 16.986C13.1836 16.986 16.9861 13.1836 16.9861 8.49303C16.9861 3.80246 13.1836 0 8.49301 0C3.80244 0 0 3.80246 0 8.49303C0 13.1836 3.80244 16.986 8.49301 16.986Z" fill="currentColor"/></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-facebook">
<a href="https://www.facebook.com/sharer/sharer.php?u=https%3A%2F%2Fslack.engineering%2Fwhat-happened-during-slacks-dnssec-rollout%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon ts-icon-facebook" width="21" height="21" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><g clip-path="url(#clip0_11_112)"><path d="M12 0C18.6274 0 24 5.37259 24 12C24 18.1352 19.3955 23.1944 13.4538 23.9121V15.667L16.7001 15.667L17.3734 12H13.4538V10.7031C13.4538 9.73417 13.6439 9.06339 14.0799 8.63483C14.5159 8.20627 15.1979 8.01993 16.1817 8.01993C16.4307 8.01993 16.6599 8.02241 16.8633 8.02736C17.1591 8.03456 17.4002 8.047 17.568 8.06467V4.74048C17.501 4.72184 17.4218 4.70321 17.3331 4.68486C17.1321 4.6433 16.8822 4.60324 16.6136 4.56806C16.0523 4.49453 15.4093 4.4423 14.9594 4.4423C13.1424 4.4423 11.7692 4.83102 10.8107 5.63619C9.65388 6.60791 9.10108 8.18622 9.10108 10.4199V12H6.62659V15.667H9.10108V23.6466C3.87432 22.3498 0 17.6277 0 12C0 5.37259 5.37259 0 12 0Z" fill="currentColor"/></g><defs><clipPath id="clip0_11_112"><rect width="24" height="24" fill="white"/></clipPath></defs></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-threads">
<a href="https://www.threads.net/intent/post?text=+https%3A%2F%2Fslack.engineering%2Fwhat-happened-during-slacks-dnssec-rollout%2F" class="ts-share-buttons-link" target="_blank">
<svg class="ts-share-buttons-icon ts-icon" width="21" viewBox="0 0 448 512" xmlns="http://www.w3.org/2000/svg"><path d="M331.5 235.7c2.2 .9 4.2 1.9 6.3 2.8c29.2 14.1 50.6 35.2 61.8 61.4c15.7 36.5 17.2 95.8-30.3 143.2c-36.2 36.2-80.3 52.5-142.6 53h-.3c-70.2-.5-124.1-24.1-160.4-70.2c-32.3-41-48.9-98.1-49.5-169.6V256v-.2C17 184.3 33.6 127.2 65.9 86.2C102.2 40.1 156.2 16.5 226.4 16h.3c70.3 .5 124.9 24 162.3 69.9c18.4 22.7 32 50 40.6 81.7l-40.4 10.8c-7.1-25.8-17.8-47.8-32.2-65.4c-29.2-35.8-73-54.2-130.5-54.6c-57 .5-100.1 18.8-128.2 54.4C72.1 146.1 58.5 194.3 58 256c.5 61.7 14.1 109.9 40.3 143.3c28 35.6 71.2 53.9 128.2 54.4c51.4-.4 85.4-12.6 113.7-40.9c32.3-32.2 31.7-71.8 21.4-95.9c-6.1-14.2-17.1-26-31.9-34.9c-3.7 26.9-11.8 48.3-24.7 64.8c-17.1 21.8-41.4 33.6-72.7 35.3c-23.6 1.3-46.3-4.4-63.9-16c-20.8-13.8-33-34.8-34.3-59.3c-2.5-48.3 35.7-83 95.2-86.4c21.1-1.2 40.9-.3 59.2 2.8c-2.4-14.8-7.3-26.6-14.6-35.2c-10-11.7-25.6-17.7-46.2-17.8H227c-16.6 0-39 4.6-53.3 26.3l-34.4-23.6c19.2-29.1 50.3-45.1 87.8-45.1h.8c62.6 .4 99.9 39.5 103.7 107.7l-.2 .2zm-156 68.8c1.3 25.1 28.4 36.8 54.6 35.3c25.6-1.4 54.6-11.4 59.5-73.2c-13.2-2.9-27.8-4.4-43.4-4.4c-4.8 0-9.6 .1-14.4 .4c-42.9 2.4-57.2 23.2-56.2 41.8l-.1 .1z"></path></svg>
</a>
</div>
<div class="ts-share-buttons-item ts-share-buttons-copy">
<a href="#" class="ts-share-buttons-link" data-share-link title="Copy link">
<svg width="21" height="21" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M10 13C10.4295 13.5741 10.9774 14.0491 11.6066 14.3929C12.2357 14.7367 12.9315 14.9411 13.6467 14.9923C14.3618 15.0435 15.0796 14.9403 15.7513 14.6897C16.4231 14.4392 17.0331 14.047 17.54 13.54L20.54 10.54C21.4508 9.59695 21.9548 8.33394 21.9434 7.02296C21.932 5.71198 21.4061 4.45791 20.4791 3.53087C19.5521 2.60383 18.298 2.07799 16.987 2.0666C15.676 2.0552 14.413 2.55918 13.47 3.46997L11.75 5.17997M14 11C13.5705 10.4259 13.0226 9.9508 12.3934 9.60704C11.7642 9.26328 11.0684 9.05886 10.3533 9.00765C9.63816 8.95643 8.92037 9.05961 8.24861 9.3102C7.57685 9.56079 6.96684 9.95291 6.45996 10.46L3.45996 13.46C2.54917 14.403 2.04519 15.666 2.05659 16.977C2.06798 18.288 2.59382 19.542 3.52086 20.4691C4.4479 21.3961 5.70197 21.922 7.01295 21.9334C8.32393 21.9447 9.58694 21.4408 10.53 20.53L12.24 18.82" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
</a>
<span class="ts-share-notification" data-share-notification>Copied!</span>
</div>
</div>
</div>
</div>
</div>
</aside>
</div>
</div>
<!-- Next and Previous Post Pagination -->
<div style="margin-top: 5rem;">
<div class="ts-container">
<div class="ts-post-pagination">
<div>
<p class="ts-section-heading">Previous Post</p>
<div class="ts-post-unfurl">
<div class="ts-post-unfurl-content">
<div>
<a href="https://slack.engineering/developing-in-the-open/">
<strong>Developing in the Open</strong>
<span aria-label="(opens in new tab)"></span>
</a>
<p class="ts-post-unfurl-text">
We use plenty of open source tools at Slack and we&#8217;ve benefited immensely from the&hellip; </p>
</div>
<div class="ts-entry__post-meta">
<div class="ts-meta-date">November 10, 2021</div>
<div class="ts-meta-reading-time"><div class="ts-meta-reading-time">3 min read</div></div>
</div>
</div>
<div class="ts-post-unfurl-img">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2021/11/elaine-casap-qgHGDbbSNm8-unsplash.jpg?w=160&amp;h=160&amp;crop=1" class="attachment-thumbnail size-thumbnail wp-post-image" alt="A bowl of tomatoes being handed from one pair of hands to another, illustrating sharing fruits of labor." decoding="async" loading="lazy" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/11/elaine-casap-qgHGDbbSNm8-unsplash.jpg?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/elaine-casap-qgHGDbbSNm8-unsplash.jpg?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2021/11/elaine-casap-qgHGDbbSNm8-unsplash.jpg?resize=110,110 110w" sizes="auto, (max-width: 160px) 100vw, 160px" /> </div>
</div>
</div>
<div>
<p class="ts-section-heading">Next Post</p>
<div class="ts-post-unfurl">
<div class="ts-post-unfurl-content">
<a href="https://slack.engineering/going-from-coder-to-slack-engineer/">
<strong>Going from Coder to Slack Engineer</strong>
<span aria-label="(opens in new tab)"></span>
</a>
<p class="ts-post-unfurl-text">
Over 70% of the files uploaded on Slack are images, and over 75% of those&hellip; </p>
<div class="ts-entry__post-meta">
<div class="ts-meta-date">December 2, 2021</div>
<div class="ts-meta-reading-time"><div class="ts-meta-reading-time">5 min read</div></div>
</div>
</div>
<div class="ts-post-unfurl-img">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2021/12/marvin-meyer-SYTO3xs06fU-unsplash.jpg?w=160&amp;h=160&amp;crop=1" class="attachment-thumbnail size-thumbnail wp-post-image" alt="Photo by Marvin Meyer on Unsplash" decoding="async" loading="lazy" srcset="https://slack.engineering/wp-content/uploads/sites/7/2021/12/marvin-meyer-SYTO3xs06fU-unsplash.jpg?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2021/12/marvin-meyer-SYTO3xs06fU-unsplash.jpg?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2021/12/marvin-meyer-SYTO3xs06fU-unsplash.jpg?resize=110,110 110w" sizes="auto, (max-width: 160px) 100vw, 160px" /> </div>
</div>
</div>
</div>
</div>
</div>
<div class="ts-entry__post-related">
<div class="ts-container">
<p class="ts-section-heading">Recommended Reading</p>
<div class="ts-entry__post-wrap">
<article class="post-17832 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-migration ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/05/cover_1.jpg?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">July 14, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/"><span>Shipyard: How We Built Slack’s Next-Generation EC2 Platform</span></a></h2><span class="ts-mention">@Archie Gunasekara</span> </div>
</div>
</article>
<article class="post-17854 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-automation-testing tag-developer-productivity tag-innovation ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/06/agent_vs_traditional.png?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="Agentic vs. traditional testing paths" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/agentic-testing-where-agents-fit-in-the-e2e-testing-stack/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">June 11, 2026</div> <div class="ts-meta-reading-time">12 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/agentic-testing-where-agents-fit-in-the-e2e-testing-stack/"><span>Agentic Testing: Where Agents Fit in the E2E Testing Stack</span></a></h2><span class="ts-mention">@Sergii Gorbachov</span> </div>
</div>
</article>
<article class="post-17812 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-backend tag-cloud-computing tag-collaboration tag-engineering tag-infrastructure tag-innovation tag-machine-learning tag-software-development ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/05/pexels-photograph-6732010.jpg?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">May 28, 2026</div> <div class="ts-meta-reading-time">17 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/"><span>Slack AI: The Path to Multi-Cloud</span></a></h2><span class="ts-mention">@Shaurya Kethireddy</span> </div>
</div>
</article>
<article class="post-17774 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-airflow tag-aws tag-big-data tag-data-engineering tag-data-infrastructure tag-security ts-entry">
<div class="ts-entry__outer">
<div class="ts-entry__inner ts-entry__thumbnail">
<div class="ts-overlay-background filter-mayfair">
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/04/SSH-Rest-CoverImage.png?w=380&amp;h=250&amp;crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="From SSH to REST_ A Security-Driven Modernization of Slack&#039;s EMR Data Pipelines" decoding="async" loading="lazy" /> </div>
<a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/" class="ts-overlay-link"></a>
</div>
<div class="ts-entry__inner ts-entry__content">
<div class="ts-entry__post-meta">
<div class="ts-meta-date">May 5, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
<h2 class="ts-entry__title"><a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/"><span>From SSH to REST: A Security-Driven Modernization of Slack&#8217;s EMR Data Pipelines</span></a></h2><span class="ts-mention">@Mahendran Vasagam</span> </div>
</div>
</article>
</div>
</div>
</div>
<!-- Arc -->
<div class="c-arc concave-up">
<div class="c-arc__upper color-blue"></div>
<div class="c-arc__lower color-white"></div>
</div>
</div>
</main>
<footer class="ts-footer">
<div class="ts-container">
<div class="ts-footer__item">
<div class="ts-footer__col ts-col-left">
<div class="ts-footer__inner">
<div class="ts-logo">
<a class="ts-footer__logo" href="https://slack.engineering/">
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
</a>
</div>
</div>
</div>
<div class="ts-footer__col ts-col-canter">
<div class="ts-footer__inner">
<nav class="ts-footer__nav">
<div class="ts-footer__nav-item">
<ul class="ts-footer__nav-inner">
<li><a href="https://slack.engineering/articles/">Articles</a></li>
<!--<li><a href="https://slack.engineering/about/">About</a></li>-->
<li><a href="https://slack.com/careers/dept/software-engineering" target="_blank">Careers</a></li>
<li><a href="https://slack.dev/blog/" target="_blank">Slack Developer Blog</a></li>
<li><a href="https://slack.com/blog/" target="_blank">The Slack Blog</a></li>
</ul>
</div>
</nav>
</div>
</div>
<div class="ts-footer__col ts-col-right">
<div class="ts-footer__inner">
<span>
<a href="https://x.com/SlackEng" target="_blank" class="ts-share-buttons-twitter">
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M9.70487 6.85148L15.289 0.5H13.9657L9.11705 6.0149L5.24443 0.5H0.777832L6.63398 8.8395L0.777832 15.5H2.10116L7.22147 9.67608L11.3112 15.5H15.7778L9.70455 6.85148H9.70487ZM7.89239 8.91297L7.29905 8.08255L2.57797 1.47476H4.61052L8.42048 6.80746L9.01383 7.63788L13.9663 14.5696H11.9338L7.89239 8.91329V8.91297Z" fill="currentcolor"></path>
</svg>
</a>
<a href="https://www.youtube.com/@Slackhq" target="_blank" class="ts-share-buttons-youtube">
<svg width="21" height="14" viewBox="0 0 21 14" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M10.463 0C19.0721 0 20.1483 0 20.1483 7C20.1483 14 19.0721 14 10.463 14C1.85397 14 0.777832 14 0.777832 7C0.777832 0 1.85397 0 10.463 0ZM8.46143 2.7998C8.10631 2.7998 7.77271 3.0798 7.77271 3.4998V10.4998C7.77271 10.9521 8.11707 11.1998 8.46143 11.1998C8.60133 11.1998 8.7197 11.1567 8.82732 11.0921L14.3694 7.59211C14.5846 7.47365 14.6923 7.26904 14.6923 6.9998C14.6923 6.73057 14.5846 6.52596 14.3694 6.4075L8.82732 2.9075C8.70894 2.83211 8.59057 2.7998 8.46143 2.7998Z" fill="currentcolor"></path>
</svg>
</a>
</span>
</div>
</div>
</div>
<br/><br/>
<div class="ts-footer__item">
<p class="ts-footer__info">
<a href="https://slack.com/terms-of-service/user" target="_blank">Terms of Service</a>
<a href="https://slack.com/trust/privacy/privacy-policy" target="_blank">Privacy Information</a>
<a href="#" class="optanon-show-settings">Cookie Preferences</a>
<a href="https://www.salesforce.com/form/other/privacy-request/?_gl=1*1yvb56x*_gcl_au*OTY0OTg5MzcuMTcyNTkxMjE5Nw..*_ga*MTMzNjk3MzQ3MC4xNzE4MDQ0MzEy*_ga_QTJQME5M5D*MTcyNjI2OTEwNy40OS4xLjE3MjYyNjkyMzAuNS4wLjA." target="_blank">Your Privacy Choices</a>
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="30" height="14"><path d="M7.4 12.8h6.8l3.1-11.6H7.4C4.2 1.2 1.6 3.8 1.6 7s2.6 5.8 5.8 5.8z" style="fill-rule:evenodd;clip-rule:evenodd;fill:#fff"/><path d="M22.6 0H7.4c-3.9 0-7 3.1-7 7s3.1 7 7 7h15.2c3.9 0 7-3.1 7-7s-3.2-7-7-7zm-21 7c0-3.2 2.6-5.8 5.8-5.8h9.9l-3.1 11.6H7.4c-3.2 0-5.8-2.6-5.8-5.8z" style="fill-rule:evenodd;clip-rule:evenodd;fill:#06f"/><path d="M24.6 4c.2.2.2.6 0 .8L22.5 7l2.2 2.2c.2.2.2.6 0 .8-.2.2-.6.2-.8 0l-2.2-2.2-2.2 2.2c-.2.2-.6.2-.8 0-.2-.2-.2-.6 0-.8L20.8 7l-2.2-2.2c-.2-.2-.2-.6 0-.8.2-.2.6-.2.8 0l2.2 2.2L23.8 4c.2-.2.6-.2.8 0z" style="fill:#fff"/><path d="M12.7 4.1c.2.2.3.6.1.8L8.6 9.8c-.1.1-.2.2-.3.2-.2.1-.5.1-.7-.1L5.4 7.7c-.2-.2-.2-.6 0-.8.2-.2.6-.2.8 0L8 8.6l3.8-4.5c.2-.2.6-.2.9 0z" style="fill:#06f"/></svg>
<br/><br/>
&copy; 2026 Slack Technologies, LLC, a Salesforce company. All rights reserved. Various trademarks held by their respective owners.
</p>
</div>
</div>
</footer>
</div>
</div>
<a href="#top" data-scroll-to-top class="ts-scroll-to-top">
<svg class="ts-icon-up" width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M5 12H19M19 12L12 5M19 12L12 19" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
<span>scroll to top</span>
</a>
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/sites/7/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/tinyspeck/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script type="text/javascript" src="https://slack.engineering/_static/??/wp-includes/js/imagesloaded.min.js,/wp-content/themes/tinyspeck/assets/src/js/site.js?m=1787604914j" ></script><script id="jetpack-stats-js-before">
_stq = window._stq || [];
_stq.push([ "view", {"v":"ext","blog":"195077423","post":"14285","tz":"-7","srv":"slack.engineering","hp":"vip","j":"1:16.1.3"} ]);
_stq.push([ "clickTrackerInit", "195077423", "14285" ]);
//# sourceURL=jetpack-stats-js-before
</script>
<script data-wp-strategy="defer" defer fetchpriority="low" id="jetpack-stats-js" src="https://stats.wp.com/e-202637.js"></script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://slack.engineering/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://slack.engineering/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>