604 lines
78 KiB
HTML
604 lines
78 KiB
HTML
<!doctype html>
|
||
<html lang="en-US">
|
||
<head>
|
||
<meta charset="UTF-8" />
|
||
<meta name="viewport" content="width=device-width, height=device-height, initial-scale=1.0, maximum-scale=1">
|
||
<meta http-equiv="X-UA-Compatible" content="ie=edge">
|
||
<link rel="profile" href="https://gmpg.org/xfn/11" />
|
||
<link rel="shortcut icon" href="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/icons/favicon.png" sizes="16x16 32x32 48x48" type="image/png">
|
||
<!-- Optanon Consent Notice start -->
|
||
<script src="https://d34u8crftukxnk.cloudfront.net/onetrust/slack.engineering/production/scripttemplates/otSDKStub.js" type="text/javascript" charset="UTF-8" data-domain-script="019872bf-9ff8-7d80-8158-f0c910bcdfad"></script>
|
||
<!-- Optanon Consent Notice end -->
|
||
<title>We’re All Just Looking for Connection | Engineering at Slack</title>
|
||
<meta name="description" content="We’ve been working to bring components of Quip’s technology into Slack with the canvas feature, while also maintaining the stand-alone Quip product. Quip’s backend, which powers both Quip and canvas, is written in Python. This is the story of a tricky bug we encountered last July and the lessons we learned along the way about…">
|
||
<meta property="og:url" content="https://slack.engineering/were-all-just-looking-for-connection/">
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:title" content="We’re All Just Looking for Connection">
|
||
<meta property="og:description" content="We’ve been working to bring components of Quip’s technology into Slack with the canvas feature, while also maintaining the stand-alone Quip product. Quip’s backend, which powers both Quip and canvas, is written in Python. This is the story of a tricky bug we encountered last July and the lessons we learned along the way about…">
|
||
<meta property="og:image" content="https://slack.engineering/wp-content/uploads/sites/7/2024/10/54040277712_ab333a09d9_b.jpg">
|
||
<meta property="og:image:width" content="6000">
|
||
<meta property="og:image:height" content="4000">
|
||
<meta name="twitter:card" content="summary_large_image">
|
||
<meta property="twitter:domain" content="slack.engineering">
|
||
<meta property="twitter:url" content="https://slack.engineering/were-all-just-looking-for-connection/">
|
||
<meta name="twitter:title" content="We’re All Just Looking for Connection">
|
||
<meta name="twitter:description" content="We’ve been working to bring components of Quip’s technology into Slack with the canvas feature, while also maintaining the stand-alone Quip product. Quip’s backend, which powers both Quip and canvas, is written in Python. This is the story of a tricky bug we encountered last July and the lessons we learned along the way about…">
|
||
<meta name="twitter:image" content="https://slack.engineering/wp-content/uploads/sites/7/2024/10/54040277712_ab333a09d9_b.jpg">
|
||
|
||
<!-- Google tag (gtag.js) -->
|
||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-590Q1VHB01"></script>
|
||
<script>
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
gtag('js', new Date());
|
||
gtag('config', 'G-590Q1VHB01');
|
||
</script>
|
||
<!-- End Google Analytics -->
|
||
<title>We’re All Just Looking for Connection – Engineering at Slack</title>
|
||
<meta name='robots' content='max-image-preview:large' />
|
||
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://slack.engineering/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fslack.engineering%2Fwere-all-just-looking-for-connection%2F" />
|
||
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://slack.engineering/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fslack.engineering%2Fwere-all-just-looking-for-connection%2F&format=xml" />
|
||
<style id="wp-img-auto-sizes-contain-inline-css">
|
||
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
|
||
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
|
||
</style>
|
||
<style id="wp-block-paragraph-inline-css">
|
||
.is-small-text{font-size:.875em}.is-regular-text{font-size:1em}.is-large-text{font-size:2.25em}.is-larger-text{font-size:3em}.has-drop-cap:not(:focus):first-letter{float:left;font-size:8.4em;font-style:normal;font-weight:100;line-height:.68;margin:.05em .1em 0 0;text-transform:uppercase}body.rtl .has-drop-cap:not(:focus):first-letter{float:none;margin-left:.1em}p.has-drop-cap.has-background{overflow:hidden}:root :where(p.has-background){padding:1.25em 2.375em}:where(p.has-text-color:not(.has-link-color)) a{color:inherit}p.has-text-align-left[style*="writing-mode:vertical-lr"],p.has-text-align-right[style*="writing-mode:vertical-rl"]{rotate:180deg}
|
||
/*# sourceURL=https://slack.engineering/wp-includes/blocks/paragraph/style.min.css */
|
||
</style>
|
||
<style id="wp-emoji-styles-inline-css">
|
||
|
||
img.wp-smiley, img.emoji {
|
||
display: inline !important;
|
||
border: none !important;
|
||
box-shadow: none !important;
|
||
height: 1em !important;
|
||
width: 1em !important;
|
||
margin: 0 0.07em !important;
|
||
vertical-align: -0.1em !important;
|
||
background: none !important;
|
||
padding: 0 !important;
|
||
}
|
||
/*# sourceURL=wp-emoji-styles-inline-css */
|
||
</style>
|
||
<style id="wp-block-library-inline-css">
|
||
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
|
||
|
||
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
|
||
</style>
|
||
<style id="classic-theme-styles-inline-css">
|
||
/*! This file is auto-generated */
|
||
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
|
||
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
|
||
</style>
|
||
|
||
<style id="global-styles-inline-css">
|
||
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
|
||
/*# sourceURL=global-styles-inline-css */
|
||
</style>
|
||
|
||
<link rel='stylesheet' id='all-css-14' href='https://slack.engineering/_static/??-eJxtjEkOgkAQRS8klopGXRjP0hQlVHoMvwjh9ja61OUf3qOlNJyTSTIqYR40gRAc+6196UCwNQhhzJNx7gV7Bnb0hyqTIjpTpqDdN1H9ko0Spcneaa/uh/6s9aRpRRH25ACxapu7UFWbAGqycc/4OF5vbXs53Q/nN25bRHg=' type='text/css' media='all' />
|
||
<script type="text/javascript" src="https://slack.engineering/_static/??-eJyFi0sOgCAQQy+kjiRGdGE8i0HEQRiUT4y3FxL3rprXvsJ91sJRlBThNEkhBQhmEUdpN1SgM+/OR+FWGRodKsgPJGFS5rLqK0n/fNFYpF+ptqj8EuUnz3ZifOB9242s0y+jLDUm" ></script><link rel="https://api.w.org/" href="https://slack.engineering/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://slack.engineering/wp-json/wp/v2/posts/17027" /> <style>img#wpstats{display:none}</style>
|
||
<link rel="icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=32" sizes="32x32" />
|
||
<link rel="icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=192" sizes="192x192" />
|
||
<link rel="apple-touch-icon" href="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=180" />
|
||
<meta name="msapplication-TileImage" content="https://slack.engineering/wp-content/uploads/sites/7/2020/05/cropped-octothrope-1.png?w=270" />
|
||
</head>
|
||
|
||
<body class="wp-singular post-template-default single single-post postid-17027 single-format-standard wp-embed-responsive wp-theme-tinyspeck">
|
||
<a class="ts-skip-link" href="#main">Skip to main content</a>
|
||
<div class="ts-site">
|
||
<div class="ts-site-inner">
|
||
<div class="ts-header-before"></div>
|
||
<header class="ts-header">
|
||
<div class="ts-container">
|
||
<div class="ts-header__inner ts-header__inner-desktop">
|
||
<div class="ts-header__col ts-col-left">
|
||
<div class="ts-logo">
|
||
<a class="ts-header__logo" href="https://slack.engineering/">
|
||
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
|
||
</a>
|
||
</div>
|
||
</div>
|
||
<div class="ts-header__col ts-col-center"></div>
|
||
<div class="ts-header__col ts-col-right">
|
||
<!--<a href="https://slack.engineering/articles/">Articles</a>
|
||
<a href="https://slack.engineering/about/">About</a>-->
|
||
<span class="ts-header__search-toggle" role="button" data-search-toggle="open">
|
||
<span class="ts-header__search-toggle-label">Search</span>
|
||
<svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
|
||
</span>
|
||
</div>
|
||
</div>
|
||
<div class="ts-header__inner ts-header__inner-mobile">
|
||
<div class="ts-header__col ts-col-left"></div>
|
||
<div class="ts-header__col ts-col-center">
|
||
<div class="ts-logo">
|
||
<a class="ts-header__logo" href="https://slack.engineering/">
|
||
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
|
||
</a>
|
||
</div>
|
||
</div>
|
||
<div class="ts-header__col ts-col-right">
|
||
<span class="ts-header__search-toggle" role="button" data-search-toggle="open">
|
||
<span class="ts-header__search-toggle-label">Search</span>
|
||
<svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="ts-search">
|
||
<div class="ts-container">
|
||
<div class="ts-search__close">
|
||
<div class="ts-search__close-inner" role="button" data-search-toggle="close">
|
||
<span class="ts-search__close-label">Close</span>
|
||
<svg class="ts-search__close-icon ts-icon ts-icon-x" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M18 6L6 18M6 6L18 18" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
|
||
</div>
|
||
</div>
|
||
<div class="ts-search__wrapper">
|
||
<form role="search" method="get" class="ts-search__nav-form" action="https://slack.engineering/">
|
||
<div class="ts-search__group">
|
||
<input class="ts-search__input" type="search" required value="" name="s" placeholder="What are you looking for?">
|
||
<button class="ts-search__submit">Search</button>
|
||
</div>
|
||
</form>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</header>
|
||
<main id="main" class="ts-site-primary">
|
||
<div class="ts-site-content ts-sidebar-enabled ts-singular-thumbnail-enabled">
|
||
<div class="ts-container">
|
||
<div class="ts-main-content">
|
||
|
||
<div class="ts-content-area">
|
||
<div class="ts-entry__header ts-entry__header-large">
|
||
<div class="ts-entry__header-inner">
|
||
<div class="ts-entry__header-info">
|
||
<div class="ts-entry__post-meta-wrapper">
|
||
<div class="ts-entry__post-meta">
|
||
<div class="ts-meta-date">October 10, 2024</div>
|
||
<div class="ts-meta-date">
|
||
Updated: October 11, 2024 </div>
|
||
<div class="ts-meta-reading-time">9 min read</div> </div>
|
||
</div>
|
||
<h1 class="ts-jumbo"><span>We’re All Just Looking for Connection</span></h1> <p class="ts-hero-paragraph"></p>
|
||
<div class="c-article__author__meta"><div class="c-article__author"><div class="c-article__author__avatar"><img src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/412906745_6867753886627663_3706485272762294854_n.jpg"/></div><div class="c-article__author__info"><span class="c-article__author__name">Brett Wines</span><span class="c-article__author__title">Senior Staff Engineer, Infrastructure</span></div></div></div> </div>
|
||
<figure class="ts-entry__post-media post-media filter-mayfair">
|
||
<img src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/54040277712_ab333a09d9_b.jpg?w=1020" class="attachment-post-thumbnail size-post-thumbnail wp-post-image" alt="" decoding="async" fetchpriority="high" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/10/54040277712_ab333a09d9_b.jpg 1024w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/54040277712_ab333a09d9_b.jpg?resize=640,360 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/54040277712_ab333a09d9_b.jpg?resize=768,432 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/54040277712_ab333a09d9_b.jpg?resize=380,214 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/54040277712_ab333a09d9_b.jpg?resize=800,450 800w" sizes="(max-width: 1020px) 100vw, 1020px" /> </figure>
|
||
</div>
|
||
</div>
|
||
<div class="ts-entry__wrap">
|
||
<div class="ts-entry__container">
|
||
|
||
<aside class="ts-widget-area ts-sidebar__area">
|
||
<div class="ts-sidebar__inner">
|
||
<!-- Search -->
|
||
<div class="widget widget_search">
|
||
<p class="ts-section-heading">Search</p>
|
||
|
||
<form role="search" method="get" class="ts-search__form" action="https://slack.engineering/">
|
||
<div class="ts-search__container">
|
||
<input required class="ts-search__input" type="search" value="" name="s" placeholder="">
|
||
<button class="ts-search__submit"><svg class="ts-icon ts-icon-search" width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M21 21L16.7 16.7M19 11C19 15.4183 15.4183 19 11 19C6.58172 19 3 15.4183 3 11C3 6.58172 6.58172 3 11 3C15.4183 3 19 6.58172 19 11Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg></button>
|
||
</div>
|
||
</form>
|
||
</div>
|
||
|
||
<!-- Latest Posts -->
|
||
<div class="widget">
|
||
<p class="ts-section-heading">Latest Posts</p>
|
||
<div class="widget-body ts-widget-posts">
|
||
<ul>
|
||
<li class="ts-post-item">
|
||
<article class="post-17832 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-migration ts-entry">
|
||
<div class="ts-entry__outer">
|
||
<div class="ts-entry__inner ts-entry__content">
|
||
<div class="ts-entry__post-meta ">
|
||
<div class="ts-meta-date">July 14, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
|
||
<h2 class="ts-entry__title">
|
||
<a href="https://slack.engineering/shipyard-how-we-built-slacks-next-generation-ec2-platform/">Shipyard: How We Built Slack’s Next-Generation EC2 Platform</a>
|
||
</h2>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
</li>
|
||
<li class="ts-post-item">
|
||
<article class="post-17854 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-automation-testing tag-developer-productivity tag-innovation ts-entry">
|
||
<div class="ts-entry__outer">
|
||
<div class="ts-entry__inner ts-entry__content">
|
||
<div class="ts-entry__post-meta ">
|
||
<div class="ts-meta-date">June 11, 2026</div> <div class="ts-meta-reading-time">12 min read</div> </div>
|
||
<h2 class="ts-entry__title">
|
||
<a href="https://slack.engineering/agentic-testing-where-agents-fit-in-the-e2e-testing-stack/">Agentic Testing: Where Agents Fit in the E2E Testing Stack</a>
|
||
</h2>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
</li>
|
||
<li class="ts-post-item">
|
||
<article class="post-17812 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-backend tag-cloud-computing tag-collaboration tag-engineering tag-infrastructure tag-innovation tag-machine-learning tag-software-development ts-entry">
|
||
<div class="ts-entry__outer">
|
||
<div class="ts-entry__inner ts-entry__content">
|
||
<div class="ts-entry__post-meta ">
|
||
<div class="ts-meta-date">May 28, 2026</div> <div class="ts-meta-reading-time">17 min read</div> </div>
|
||
<h2 class="ts-entry__title">
|
||
<a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/">Slack AI: The Path to Multi-Cloud</a>
|
||
</h2>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
</li>
|
||
<li class="ts-post-item">
|
||
<article class="post-17774 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-airflow tag-aws tag-big-data tag-data-engineering tag-data-infrastructure tag-security ts-entry">
|
||
<div class="ts-entry__outer">
|
||
<div class="ts-entry__inner ts-entry__content">
|
||
<div class="ts-entry__post-meta ">
|
||
<div class="ts-meta-date">May 5, 2026</div> <div class="ts-meta-reading-time">15 min read</div> </div>
|
||
<h2 class="ts-entry__title">
|
||
<a href="https://slack.engineering/from-ssh-to-rest-a-security-driven-modernization-of-slacks-emr-data-pipelines/">From SSH to REST: A Security-Driven Modernization of Slack’s EMR Data Pipelines</a>
|
||
</h2>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
</li>
|
||
<li class="ts-post-item">
|
||
<article class="post-17752 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-development tag-security tag-software-engineering ts-entry">
|
||
<div class="ts-entry__outer">
|
||
<div class="ts-entry__inner ts-entry__content">
|
||
<div class="ts-entry__post-meta ">
|
||
<div class="ts-meta-date">April 13, 2026</div> <div class="ts-meta-reading-time">14 min read</div> </div>
|
||
<h2 class="ts-entry__title">
|
||
<a href="https://slack.engineering/managing-context-in-long-run-agentic-applications/">Managing context in long-run agentic applications</a>
|
||
</h2>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Archives -->
|
||
<div class="widget">
|
||
<p class="ts-section-heading">Archives</p>
|
||
<ul class="wp-block-archives-list wp-block-archives">
|
||
<li><a href='https://slack.engineering/2026/'>2026</a> (7)</li>
|
||
<li><a href='https://slack.engineering/2025/'>2025</a> (9)</li>
|
||
<li><a href='https://slack.engineering/2024/'>2024</a> (19)</li>
|
||
<li><a href='https://slack.engineering/2023/'>2023</a> (16)</li>
|
||
<li><a href='https://slack.engineering/2022/'>2022</a> (21)</li>
|
||
<li><a href='https://slack.engineering/2021/'>2021</a> (24)</li>
|
||
<li><a href='https://slack.engineering/2020/'>2020</a> (26)</li>
|
||
<li><a href='https://slack.engineering/2019/'>2019</a> (21)</li>
|
||
<li><a href='https://slack.engineering/2018/'>2018</a> (11)</li>
|
||
<li><a href='https://slack.engineering/2017/'>2017</a> (21)</li>
|
||
<li><a href='https://slack.engineering/2016/'>2016</a> (19)</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</aside>
|
||
|
||
<div class="ts-entry__content-wrap">
|
||
<div class="ts-single-post ts-wysiwyg">
|
||
|
||
<p class="wp-block-paragraph">We’ve been working to bring components of Quip’s technology into Slack with the <a href="https://slack.com/features/canvas">canvas</a> feature, while also maintaining the stand-alone Quip product. Quip’s backend, which powers both Quip and canvas, is written in Python. This is the story of a tricky bug we encountered last July and the lessons we learned along the way about being careful with TCP state. We hope that showing you how we tackled our bug helps you avoid — or find — similar bugs in the future!<br /><br />Our adventure began with a spike in <code>EOFError</code> errors during SQL queries. The errors were distributed across multiple services and multiple database hosts:</p>
|
||
<p><img decoding="async" width="1035" height="804" class="alignnone size-medium wp-image-17174" src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image.jpg?w=640" alt="A timeseries graph showing distribution of errors across DB group types." srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image.jpg 1035w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image.jpg?resize=640,497 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image.jpg?resize=768,597 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image.jpg?resize=380,295 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image.jpg?resize=800,621 800w" sizes="(max-width: 1035px) 100vw, 1035px" /></p>
|
||
<h3><span style="color: #17181e;font-family: var(--headings__FontFamily);font-size: var(--h2__FontSize);font-weight: 600">Investigation</span></h3>
|
||
<p><br />The stacktrace showed an <code>asyncio.IncompleteReadError</code>, which we translate to <code>EOFError</code>, when reading the response from the database:</p>
|
||
<pre><code class="language-c">File "core/mysql.py", line 299, in __read_result_set
|
||
header = await self.__read_packet(timeout=timeout)
|
||
File "core/mysql.py", line 532, in __read_packet
|
||
header = await self.conn.read_exactly(4, timeout=timeout)
|
||
File "core/runtime_asyncio.py", line 1125, in read_exactly
|
||
raise EOFError() from None</code></pre>
|
||
<p>Here’s the relevant code, which had not been touched recently, along with some associated baffled commentary:</p>
|
||
<pre><code class="language-python">async def _perform_query_locked(...) -> core.sql.Result:
|
||
...
|
||
if not self.conn.is_connected():
|
||
await self.__connect(...)
|
||
await self.__send_command(...)
|
||
result_set = await self.__read_result_set(...) # <-- EOFError</code></pre>
|
||
<p><img decoding="async" width="1114" height="203" class="alignnone size-medium wp-image-17175" src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image2.jpg?w=640" alt="A Github comment from Brett: "wow, insane, before every query we check for EOFError preemptively and reconnect if needed, but the stacktrace shows us as failing in the call to __read_result_set(), so we must've already reconnected..."" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image2.jpg 1114w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image2.jpg?resize=640,117 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image2.jpg?resize=768,140 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image2.jpg?resize=380,69 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image2.jpg?resize=800,146 800w" sizes="(max-width: 1114px) 100vw, 1114px" /><br /><br />There are a few places where we close our connection to the database, e.g. if we see one of a certain set of errors. One initial hypothesis was that another Python coroutine closed the connection for its own reasons, after we issued the query but before we read the response. We quickly discarded that theory since connection access is protected with an in-memory lock, which is acquired at a higher level in the code. Since that would be impossible, we reasoned that the other side of the connection must have closed on us.<br /><br />This theory was bolstered when we found that our database proxies closed a bunch of connections at the time of the event. The closes were distributed independent of database host, which was consistent with the incident’s surface area. The proxy-close metric, <code>ClientConnectionsClosed</code>, represents closes initiated by either side, but we have a separate metric for when we initiate the close, and there was no increase in activity from us at the time. The timing here was extremely suspicious, although it still didn’t quite make sense why this should result in such a high rate of errors on the response read since we had just ensured connection state.</p>
|
||
<p><img loading="lazy" decoding="async" width="309" height="267" class="size-medium wp-image-17176 alignleft" src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image3.jpg?w=309" alt="A graph of the ClientConnectionsClosed metrics showing distinct spikes." /><br /><br /><img loading="lazy" decoding="async" width="3356" height="1444" class="alignnone size-medium wp-image-17177" src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image4.jpg?w=640" alt="A timeseries graph showing distribution of closes across proxies." srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image4.jpg 3356w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image4.jpg?resize=640,275 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image4.jpg?resize=768,330 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image4.jpg?resize=1280,551 1280w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image4.jpg?resize=1536,661 1536w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image4.jpg?resize=2048,881 2048w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image4.jpg?resize=380,164 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image4.jpg?resize=800,344 800w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Image4.jpg?resize=1160,499 1160w" sizes="auto, (max-width: 3356px) 100vw, 3356px" /><br /><br />Something clearly wasn’t adding up, so it was time to dig deeper. Suspecting the <code>if not self.conn.is_connected():</code> line, we looked at our implementation for <code>AsyncioConnection</code>:</p>
|
||
<pre><code class="language-python">class AsyncioConnection(core.runtime.Connection[memoryview]):
|
||
def closed(self) -> bool:
|
||
return not self.writer or self.writer.is_closing()
|
||
|
||
def is_connected(self) -> bool:
|
||
return self.connected</code></pre>
|
||
<p>Both of these functions turned out to be incorrect. First, let’s look at <code>closed()</code>. Each Asyncio connection has a <a href="https://docs.python.org/3/library/asyncio-stream.html#streamreader"><code>StreamReader</code></a> and a <a href="https://docs.python.org/3/library/asyncio-stream.html#streamwriter"><code>StreamWriter</code></a>. Our code suggested that the closed state is only a function of the writer, which is controlled by the application server, not the reader. This seemed suspiciously incomplete to us since we expected only the reader to be aware of whether the other side closed the connection. We proved this with a unit test:</p>
|
||
<pre><code class="language-python">async def test_reader_at_eof_vs_writer_is_closing(self):
|
||
conn = await self.create_connection()
|
||
|
||
# Ask the unit test's server to close
|
||
await conn.write(self.encode_command("/quit"))
|
||
|
||
# Don't read. Still open since we haven't seen the
|
||
# response yet
|
||
self.assertFalse(conn.writer.is_closing())
|
||
|
||
# Read, then sleep(0) since it requires another run
|
||
# through the scheduler loop so the stream can detect
|
||
# the zero read/eof
|
||
response = await conn.read_until(self.io_suffix)
|
||
await core.runtime.sleep(0)
|
||
|
||
self.assertTrue(conn.reader.at_eof()) # passes
|
||
self.assertTrue(conn.writer.is_closing()) # fails</code></pre>
|
||
<p>Next, let’s look at <code>is_connected()</code>. First, it’s not a function of <code>closed()</code>, so the two have the potential to drift. Second, it could return a false positive: the only place where we set the instance variable <code>self.connected</code> to false is when the application server closes the connection, so similar to <code>closed()</code> it’s unaware of the reader being in the EOF state.<br /><br />We decided to log a metric so we could compare <code>self.connected</code> to <code>not self.closed()</code> to see exactly how much they drift. We found six more bugs, which led to one coworker coining the delightful phrase “when you go picking up rocks, you might find things under those rocks,” which we immediately adopted:<br /><br /><img loading="lazy" decoding="async" width="3282" height="448" class="alignnone size-medium wp-image-17194" src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.39.40 PM.png?w=640" alt="" data-wp-editing="1" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.39.40 PM.png 3282w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.39.40 PM.png?resize=640,87 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.39.40 PM.png?resize=768,105 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.39.40 PM.png?resize=1280,175 1280w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.39.40 PM.png?resize=1536,210 1536w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.39.40 PM.png?resize=2048,280 2048w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.39.40 PM.png?resize=380,52 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.39.40 PM.png?resize=800,109 800w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.39.40 PM.png?resize=1160,158 1160w" sizes="auto, (max-width: 3282px) 100vw, 3282px" /></p>
|
||
<ol>
|
||
<li><span style="font-family: var(--content-text__FontFamily);font-size: var(--content-text__FontSize)"><code>is_connected()</code> could also be a false negative. We saw that the false negatives only occurred on the services that maintain websocket connections to clients. The only place we set <code>self.connected</code> to true is when the application server initiates the connection, so it’s never set to true when it’s initiated by the client.</span></li>
|
||
<li><span style="font-family: var(--content-text__FontFamily);font-size: var(--content-text__FontSize)">There is a check we perform upon connection lock release to see if there is any data on the connection we haven’t read. It was incorrect to perform this check in some circumstances on websocket connections since it’s expected that the client can send data at any point in time.</span></li>
|
||
<li>Our HTTP client pool could contain closed clients.</li>
|
||
<li>We didn’t correctly handle exceptions during reconnect.</li>
|
||
<li>We found another place where we should be removing connections from the pool.</li>
|
||
<li><span style="font-family: var(--content-text__FontFamily);font-size: var(--content-text__FontSize)">The Redis-specific cleanup we thought we would always do upon connection close wasn’t happening when the close wasn’t initiated by the application server.</span></li>
|
||
</ol>
|
||
<h3><span style="color: #17181e;font-family: var(--headings__FontFamily);font-size: var(--h2__FontSize);font-weight: 600">Resolution</span></h3>
|
||
<p>At this point we had all the pieces of the puzzle and were able to understand the sequence of events:</p>
|
||
<ol>
|
||
<li>The proxy closed database connections on us. After communicating with AWS we learned that they introduced behavior where the proxy closes connections after 24h even if it isn’t idle, so the closes were tied to our daily release, which restarts all of our servers.</li>
|
||
<li><span style="font-family: var(--content-text__FontFamily);font-size: var(--content-text__FontSize)">We failed to recognize the connections were closed since the close was not initiated by us, and as a result we did not reconnect prior to issuing SQL queries.</span></li>
|
||
<li><span style="font-family: var(--content-text__FontFamily);font-size: var(--content-text__FontSize)">We’d issue a query, then attempt to read the response, but the reader would be in the EOF state from the close, so it would raise </span><code style="font-size: var(--content-text__FontSize)">EOFError</code><span style="font-family: var(--content-text__FontFamily);font-size: var(--content-text__FontSize)">.</span></li>
|
||
</ol>
|
||
<p><span style="font-family: var(--content-text__FontFamily);font-size: var(--content-text__FontSize)">We deployed the fix and saw the near-total reduction in </span><code style="font-size: var(--content-text__FontSize)">EOFError</code><span style="font-family: var(--content-text__FontFamily);font-size: var(--content-text__FontSize)"> during SQL queries:</span></p>
|
||
<p><img loading="lazy" decoding="async" width="2314" height="1924" class="alignnone size-medium wp-image-17179" src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.13.11 PM.png?w=640" alt="Three Kibana screenshots showing a near-total reduction in EOFError rate" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.13.11 PM.png 2314w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.13.11 PM.png?resize=640,532 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.13.11 PM.png?resize=768,639 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.13.11 PM.png?resize=1280,1064 1280w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.13.11 PM.png?resize=1536,1277 1536w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.13.11 PM.png?resize=2048,1703 2048w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.13.11 PM.png?resize=380,316 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.13.11 PM.png?resize=800,665 800w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.13.11 PM.png?resize=1160,964 1160w" sizes="auto, (max-width: 2314px) 100vw, 2314px" /></p>
|
||
<p>As we deployed the fix for the case where connections are initiated by the client, the false negatives disappeared entirely, too:</p>
|
||
<p><img loading="lazy" decoding="async" width="3356" height="1572" class="alignnone size-medium wp-image-17178" src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.08.57 PM.png?w=640" alt="A timeseries graph showing a reduction in the discrepancy metric." srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.08.57 PM.png 3356w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.08.57 PM.png?resize=640,300 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.08.57 PM.png?resize=768,360 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.08.57 PM.png?resize=1280,600 1280w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.08.57 PM.png?resize=1536,719 1536w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.08.57 PM.png?resize=2048,959 2048w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.08.57 PM.png?resize=380,178 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.08.57 PM.png?resize=800,375 800w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-9.08.57 PM.png?resize=1160,543 1160w" sizes="auto, (max-width: 3356px) 100vw, 3356px" /></p>
|
||
<h3><img decoding="async" class="alignnone size-medium wp-image-17049" style="font-family: var(--content-text__FontFamily);font-size: var(--content-text__FontSize)" src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/image8_43fdd2.png?w=640" alt="" /><span style="color: #17181e;font-family: var(--headings__FontFamily);font-size: var(--h2__FontSize);font-weight: 600">The Asyncio Migration</span></h3>
|
||
<p><span style="font-family: var(--content-text__FontFamily);font-size: var(--content-text__FontSize)">The primary impact of this bug fix, however, wasn’t even the improved handling of connection closes. The primary impact was the unblocking of the Python runtime migration.</span></p>
|
||
<p><br />The migration project began in 2020 and was from our custom runtime — we were early adopters of Python 3 — to <a href="https://docs.python.org/3/library/asyncio.html">asyncio</a>, the standard Python IO library introduced in 3.4. The project ran smoothly and was enabled in all environments with just one exception: one service on our largest single-tenant cluster, since the customer would sporadically experience timeouts when exporting spreadsheets to PDF. The issue only occurred at times of peak load and was difficult to reproduce, so the project stalled.<br /><br />This PDF export handler makes an RPC request to another service that handles the PDF generation itself. That request goes over an existing connection from a connection pool, if one exists. If the RPC service is overloaded, it may close that connection, and if so we would fail to recognize that. The handler would fire off the RPC request, but the other side wouldn’t be listening, so that request would hang on write and eventually time out since the buffer would never get drained by the RPC service. For small exports we expect it would have raised an <code>EOFError</code> immediately instead of hanging since it wouldn’t completely fill the buffer.<br /><br />After launching the connection state fixes, we re-enabled asyncio on the final service, and confirmed that the errors didn’t recur. The asyncio project resumed, entered a very satisfying phase of deleting code in our custom runtime scheduler, and concluded shortly after.</p>
|
||
<h3><span style="color: #17181e;font-family: var(--headings__FontFamily);font-size: var(--h2__FontSize);font-weight: 600">Conclusion</span></h3>
|
||
<p>This is tricky to get right! As we see from <a href="https://man7.org/linux/man-pages/man3/shutdown.3p.html">https://man7.org/linux/man-pages/man3/shutdown.3p.html</a> a socket can be in a half-shutdown state. We had expected <a href="https://docs.python.org/3/library/asyncio-stream.html#asyncio.StreamWriter.is_closing">from the documentation</a> that <code>StreamWriter.is_closing()</code> would encompass that half-shutdown state, but we should have been more careful:</p>
|
||
<p><em>Return True if the stream is closed or in the process of being closed.</em><br /><br />In fact, both the <a href="https://github.com/python/cpython/blob/v3.7.16/Lib/asyncio/selector_events.py#L658">default event loop</a> and <a href="https://github.com/MagicStack/uvloop/blob/v0.20.0/uvloop/handles/basetransport.pyx#L256">uvloop</a> contain the same behavior as we did, where the closing state is only set upon client-initiated close. This explains why the unit test mentioned before failed both with uvloop enabled and disabled and why it was insufficient to depend only on <code>StreamWriter.is_closing()</code>.<br /><br />When the other side of a TCP connection sends <code>FIN</code>, we enter the <code>CLOSE WAIT</code> state. But in that state we can still attempt to read for as long as we like. We’ll only exit that state when we tell the OS to close or shut down the socket.</p>
|
||
<p><img loading="lazy" decoding="async" width="1970" height="1540" class="alignnone size-medium wp-image-17030" src="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-8.49.18 PM.png?w=640" alt="" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-8.49.18 PM.png 1970w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-8.49.18 PM.png?resize=640,500 640w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-8.49.18 PM.png?resize=768,600 768w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-8.49.18 PM.png?resize=1280,1001 1280w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-8.49.18 PM.png?resize=1536,1201 1536w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-8.49.18 PM.png?resize=380,297 380w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-8.49.18 PM.png?resize=800,625 800w, https://slack.engineering/wp-content/uploads/sites/7/2024/10/Screenshot-2024-10-04-at-8.49.18 PM.png?resize=1160,907 1160w" sizes="auto, (max-width: 1970px) 100vw, 1970px" /></p>
|
||
<p>Above all, my personal takeaway is to always pick up rocks[1]. In this case it could have been tempting to simply call it a day after adding <code>EOFError</code> to the list of exceptions upon which we reissue the query to a failover database. But with low-level code, it’s usually worth the extra time and effort to dig a bit deeper if something just doesn’t quite seem right. Even if things are mostly fine on the surface, bugs down there can have far-reaching effects.</p>
|
||
<p>Special thanks to Ross Cohen for his hard work and camaraderie through this rollercoaster of a bug. Couldn’t ask for a better buddy on this one!</p>
|
||
<p>If you enjoyed this post, you might also like working here. Check out our <a href="https://slack.com/careers">careers page</a>!</p>
|
||
<p>[1]<i><span style="font-weight: 400"> You might find things under those rocks!</span></i></p>
|
||
<p></p> </div>
|
||
<div class="ts-tags-section"><span class="ts-tag">#<a href="https://slack.engineering/tags/database/" rel="tag">database</a></span><span class="ts-tag">#<a href="https://slack.engineering/tags/infrastructure/" rel="tag">infrastructure</a></span><span class="ts-tag">#<a href="https://slack.engineering/tags/networking/" rel="tag">networking</a></span><span class="ts-tag">#<a href="https://slack.engineering/tags/python/" rel="tag">python</a></span></div> </div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<aside class="ts-metabar__area">
|
||
<div class="ts-metabar__inner">
|
||
<div class="ts-metabar__item">
|
||
<div class="ts-share-buttons-wrap">
|
||
<div class="ts-share-buttons-items">
|
||
<div class="ts-share-buttons-item ts-share-buttons-twitter">
|
||
<a href="https://x.com/intent/post?url=https%3A%2F%2Fslack.engineering%2Fwere-all-just-looking-for-connection%2F" class="ts-share-buttons-link" target="_blank">
|
||
<svg class="ts-share-buttons-icon ts-icon ts-icon-twitter" width="21" viewBox="0 0 72 65" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M22.6891 0H0L26.9014 35.3477L1.72089 65H13.355L32.3999 42.5731L49.3109 64.794H72L44.317 28.4191L44.366 28.4818L68.2015 0.412927H56.5674L38.8667 21.2573L22.6891 0ZM12.524 6.19052H19.5874L59.476 58.603H52.4126L12.524 6.19052Z" fill="currentColor"/></svg>
|
||
</a>
|
||
</div>
|
||
<div class="ts-share-buttons-item ts-share-buttons-linkedin">
|
||
<a href="https://www.linkedin.com/shareArticle?mini=true&url=https%3A%2F%2Fslack.engineering%2Fwere-all-just-looking-for-connection%2F" class="ts-share-buttons-link" target="_blank">
|
||
<svg class="ts-share-buttons-icon ts-icon ts-icon-linkedin" width="21" height="21" viewBox="0 0 70 70" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M15.8066 23.0188H1.17969V69.663H15.8066V23.0188Z" fill="currentColor"/><path d="M55.3732 22.0077C54.834 21.9403 54.2611 21.9066 53.6881 21.8729C45.4984 21.5359 40.8812 26.389 39.2635 28.4786C38.8254 29.0515 38.6231 29.3886 38.6231 29.3886V23.1536H24.6366V69.7978H38.6231H39.2635C39.2635 65.0458 39.2635 60.3274 39.2635 55.5754C39.2635 53.014 39.2635 50.4526 39.2635 47.8912C39.2635 44.7232 39.0276 41.3529 40.6116 38.4545C41.9597 36.0279 44.3862 34.8147 47.1161 34.8147C55.2047 34.8147 55.3732 42.1281 55.3732 42.8021C55.3732 42.8358 55.3732 42.8695 55.3732 42.8695V70H70.0001V39.5667C70.0001 29.1526 64.7088 23.0188 55.3732 22.0077Z" fill="currentColor"/><path d="M8.49301 16.986C13.1836 16.986 16.9861 13.1836 16.9861 8.49303C16.9861 3.80246 13.1836 0 8.49301 0C3.80244 0 0 3.80246 0 8.49303C0 13.1836 3.80244 16.986 8.49301 16.986Z" fill="currentColor"/></svg>
|
||
</a>
|
||
</div>
|
||
<div class="ts-share-buttons-item ts-share-buttons-facebook">
|
||
<a href="https://www.facebook.com/sharer/sharer.php?u=https%3A%2F%2Fslack.engineering%2Fwere-all-just-looking-for-connection%2F" class="ts-share-buttons-link" target="_blank">
|
||
<svg class="ts-share-buttons-icon ts-icon ts-icon-facebook" width="21" height="21" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><g clip-path="url(#clip0_11_112)"><path d="M12 0C18.6274 0 24 5.37259 24 12C24 18.1352 19.3955 23.1944 13.4538 23.9121V15.667L16.7001 15.667L17.3734 12H13.4538V10.7031C13.4538 9.73417 13.6439 9.06339 14.0799 8.63483C14.5159 8.20627 15.1979 8.01993 16.1817 8.01993C16.4307 8.01993 16.6599 8.02241 16.8633 8.02736C17.1591 8.03456 17.4002 8.047 17.568 8.06467V4.74048C17.501 4.72184 17.4218 4.70321 17.3331 4.68486C17.1321 4.6433 16.8822 4.60324 16.6136 4.56806C16.0523 4.49453 15.4093 4.4423 14.9594 4.4423C13.1424 4.4423 11.7692 4.83102 10.8107 5.63619C9.65388 6.60791 9.10108 8.18622 9.10108 10.4199V12H6.62659V15.667H9.10108V23.6466C3.87432 22.3498 0 17.6277 0 12C0 5.37259 5.37259 0 12 0Z" fill="currentColor"/></g><defs><clipPath id="clip0_11_112"><rect width="24" height="24" fill="white"/></clipPath></defs></svg>
|
||
</a>
|
||
</div>
|
||
<div class="ts-share-buttons-item ts-share-buttons-threads">
|
||
<a href="https://www.threads.net/intent/post?text=+https%3A%2F%2Fslack.engineering%2Fwere-all-just-looking-for-connection%2F" class="ts-share-buttons-link" target="_blank">
|
||
<svg class="ts-share-buttons-icon ts-icon" width="21" viewBox="0 0 448 512" xmlns="http://www.w3.org/2000/svg"><path d="M331.5 235.7c2.2 .9 4.2 1.9 6.3 2.8c29.2 14.1 50.6 35.2 61.8 61.4c15.7 36.5 17.2 95.8-30.3 143.2c-36.2 36.2-80.3 52.5-142.6 53h-.3c-70.2-.5-124.1-24.1-160.4-70.2c-32.3-41-48.9-98.1-49.5-169.6V256v-.2C17 184.3 33.6 127.2 65.9 86.2C102.2 40.1 156.2 16.5 226.4 16h.3c70.3 .5 124.9 24 162.3 69.9c18.4 22.7 32 50 40.6 81.7l-40.4 10.8c-7.1-25.8-17.8-47.8-32.2-65.4c-29.2-35.8-73-54.2-130.5-54.6c-57 .5-100.1 18.8-128.2 54.4C72.1 146.1 58.5 194.3 58 256c.5 61.7 14.1 109.9 40.3 143.3c28 35.6 71.2 53.9 128.2 54.4c51.4-.4 85.4-12.6 113.7-40.9c32.3-32.2 31.7-71.8 21.4-95.9c-6.1-14.2-17.1-26-31.9-34.9c-3.7 26.9-11.8 48.3-24.7 64.8c-17.1 21.8-41.4 33.6-72.7 35.3c-23.6 1.3-46.3-4.4-63.9-16c-20.8-13.8-33-34.8-34.3-59.3c-2.5-48.3 35.7-83 95.2-86.4c21.1-1.2 40.9-.3 59.2 2.8c-2.4-14.8-7.3-26.6-14.6-35.2c-10-11.7-25.6-17.7-46.2-17.8H227c-16.6 0-39 4.6-53.3 26.3l-34.4-23.6c19.2-29.1 50.3-45.1 87.8-45.1h.8c62.6 .4 99.9 39.5 103.7 107.7l-.2 .2zm-156 68.8c1.3 25.1 28.4 36.8 54.6 35.3c25.6-1.4 54.6-11.4 59.5-73.2c-13.2-2.9-27.8-4.4-43.4-4.4c-4.8 0-9.6 .1-14.4 .4c-42.9 2.4-57.2 23.2-56.2 41.8l-.1 .1z"></path></svg>
|
||
</a>
|
||
</div>
|
||
<div class="ts-share-buttons-item ts-share-buttons-copy">
|
||
<a href="#" class="ts-share-buttons-link" data-share-link title="Copy link">
|
||
<svg width="21" height="21" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M10 13C10.4295 13.5741 10.9774 14.0491 11.6066 14.3929C12.2357 14.7367 12.9315 14.9411 13.6467 14.9923C14.3618 15.0435 15.0796 14.9403 15.7513 14.6897C16.4231 14.4392 17.0331 14.047 17.54 13.54L20.54 10.54C21.4508 9.59695 21.9548 8.33394 21.9434 7.02296C21.932 5.71198 21.4061 4.45791 20.4791 3.53087C19.5521 2.60383 18.298 2.07799 16.987 2.0666C15.676 2.0552 14.413 2.55918 13.47 3.46997L11.75 5.17997M14 11C13.5705 10.4259 13.0226 9.9508 12.3934 9.60704C11.7642 9.26328 11.0684 9.05886 10.3533 9.00765C9.63816 8.95643 8.92037 9.05961 8.24861 9.3102C7.57685 9.56079 6.96684 9.95291 6.45996 10.46L3.45996 13.46C2.54917 14.403 2.04519 15.666 2.05659 16.977C2.06798 18.288 2.59382 19.542 3.52086 20.4691C4.4479 21.3961 5.70197 21.922 7.01295 21.9334C8.32393 21.9447 9.58694 21.4408 10.53 20.53L12.24 18.82" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
|
||
</a>
|
||
<span class="ts-share-notification" data-share-notification>Copied!</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</aside>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Next and Previous Post Pagination -->
|
||
<div style="margin-top: 5rem;">
|
||
<div class="ts-container">
|
||
<div class="ts-post-pagination">
|
||
<div>
|
||
<p class="ts-section-heading">Previous Post</p>
|
||
<div class="ts-post-unfurl">
|
||
<div class="ts-post-unfurl-content">
|
||
<div>
|
||
<a href="https://slack.engineering/driving-a-project-intern-edition/">
|
||
<strong>Driving a Project: Intern Edition</strong>
|
||
<span aria-label="(opens in new tab)"></span>
|
||
</a>
|
||
<p class="ts-post-unfurl-text">
|
||
After a lot of hard work, you’ve landed that coveted internship. Now comes the next… </p>
|
||
</div>
|
||
<div class="ts-entry__post-meta">
|
||
<div class="ts-meta-date">September 23, 2024</div>
|
||
<div class="ts-meta-reading-time"><div class="ts-meta-reading-time">7 min read</div></div>
|
||
</div>
|
||
</div>
|
||
<div class="ts-post-unfurl-img">
|
||
<img src="https://slack.engineering/wp-content/uploads/sites/7/2024/09/caleb-whiting-bau67pJBPcA-unsplash.jpg?w=160&h=160&crop=1" class="attachment-thumbnail size-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/09/caleb-whiting-bau67pJBPcA-unsplash.jpg?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/caleb-whiting-bau67pJBPcA-unsplash.jpg?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2024/09/caleb-whiting-bau67pJBPcA-unsplash.jpg?resize=110,110 110w" sizes="auto, (max-width: 160px) 100vw, 160px" /> </div>
|
||
</div>
|
||
</div>
|
||
|
||
<div>
|
||
<p class="ts-section-heading">Next Post</p>
|
||
<div class="ts-post-unfurl">
|
||
<div class="ts-post-unfurl-content">
|
||
<a href="https://slack.engineering/empowering-engineers-with-ai/">
|
||
<strong>Empowering Engineers with AI</strong>
|
||
<span aria-label="(opens in new tab)"></span>
|
||
</a>
|
||
<p class="ts-post-unfurl-text">
|
||
Background and motivation In the fast-paced world of software development, having the right tools can… </p>
|
||
<div class="ts-entry__post-meta">
|
||
<div class="ts-meta-date">November 8, 2024</div>
|
||
<div class="ts-meta-reading-time"><div class="ts-meta-reading-time">9 min read</div></div>
|
||
</div>
|
||
</div>
|
||
<div class="ts-post-unfurl-img">
|
||
<img src="https://slack.engineering/wp-content/uploads/sites/7/2024/11/Screenshot-2024-11-07-at-4.09.53 PM.png?w=160&h=160&crop=1" class="attachment-thumbnail size-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" srcset="https://slack.engineering/wp-content/uploads/sites/7/2024/11/Screenshot-2024-11-07-at-4.09.53 PM.png?resize=160,160 160w, https://slack.engineering/wp-content/uploads/sites/7/2024/11/Screenshot-2024-11-07-at-4.09.53 PM.png?resize=80,80 80w, https://slack.engineering/wp-content/uploads/sites/7/2024/11/Screenshot-2024-11-07-at-4.09.53 PM.png?resize=110,110 110w" sizes="auto, (max-width: 160px) 100vw, 160px" /> </div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="ts-entry__post-related">
|
||
<div class="ts-container">
|
||
<p class="ts-section-heading">Recommended Reading</p>
|
||
<div class="ts-entry__post-wrap">
|
||
<article class="post-17812 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-aws tag-backend tag-cloud-computing tag-collaboration tag-engineering tag-infrastructure tag-innovation tag-machine-learning tag-software-development ts-entry">
|
||
<div class="ts-entry__outer">
|
||
<div class="ts-entry__inner ts-entry__thumbnail">
|
||
<div class="ts-overlay-background filter-mayfair">
|
||
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/05/pexels-photograph-6732010.jpg?w=380&h=250&crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
|
||
<a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/" class="ts-overlay-link"></a>
|
||
</div>
|
||
<div class="ts-entry__inner ts-entry__content">
|
||
<div class="ts-entry__post-meta">
|
||
<div class="ts-meta-date">May 28, 2026</div> <div class="ts-meta-reading-time">17 min read</div> </div>
|
||
<h2 class="ts-entry__title"><a href="https://slack.engineering/slack-ai-the-path-to-multi-cloud/"><span>Slack AI: The Path to Multi-Cloud</span></a></h2><span class="ts-mention">@Shaurya Kethireddy</span> </div>
|
||
</div>
|
||
</article>
|
||
<article class="post-17732 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-golang tag-infrastructure tag-networking tag-observability tag-open-source ts-entry">
|
||
<div class="ts-entry__outer">
|
||
<div class="ts-entry__inner ts-entry__thumbnail">
|
||
<div class="ts-overlay-background filter-mayfair">
|
||
<img src="https://slack.engineering/wp-content/uploads/sites/7/2026/03/cover.png?w=380&h=250&crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="" decoding="async" loading="lazy" /> </div>
|
||
<a href="https://slack.engineering/from-custom-to-open-scalable-network-probing-and-http-3-readiness-with-prometheus/" class="ts-overlay-link"></a>
|
||
</div>
|
||
<div class="ts-entry__inner ts-entry__content">
|
||
<div class="ts-entry__post-meta">
|
||
<div class="ts-meta-date">March 31, 2026</div> <div class="ts-meta-reading-time">5 min read</div> </div>
|
||
<h2 class="ts-entry__title"><a href="https://slack.engineering/from-custom-to-open-scalable-network-probing-and-http-3-readiness-with-prometheus/"><span>From Custom to Open: Scalable Network Probing and HTTP/3 Readiness with Prometheus</span></a></h2><span class="ts-mention">@Slack Engineering</span> </div>
|
||
</div>
|
||
</article>
|
||
<article class="post-17591 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-build-performance tag-caching tag-ci-cd tag-developer-experience tag-devops tag-python ts-entry">
|
||
<div class="ts-entry__outer">
|
||
<div class="ts-entry__inner ts-entry__thumbnail">
|
||
<div class="ts-overlay-background filter-mayfair">
|
||
<img src="https://slack.engineering/wp-content/uploads/sites/7/2025/10/DevXP-Council-Presentation6-e1759869480617.png?w=380&h=250&crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="An example service build graph. It shows a set of Python files building into a backend artifact and a set of TypeScript files building into a frontend artifact." decoding="async" loading="lazy" /> </div>
|
||
<a href="https://slack.engineering/build-better-software-to-build-software-better/" class="ts-overlay-link"></a>
|
||
</div>
|
||
<div class="ts-entry__inner ts-entry__content">
|
||
<div class="ts-entry__post-meta">
|
||
<div class="ts-meta-date">November 6, 2025</div> <div class="ts-meta-reading-time">18 min read</div> </div>
|
||
<h2 class="ts-entry__title"><a href="https://slack.engineering/build-better-software-to-build-software-better/"><span>Build better software to build software better</span></a></h2><span class="ts-mention">@David Reed</span> </div>
|
||
</div>
|
||
</article>
|
||
<article class="post-17513 post type-post status-publish format-standard has-post-thumbnail category-uncategorized tag-automation tag-ci-cd tag-deployment tag-engineering tag-incident-response tag-infrastructure tag-observability ts-entry">
|
||
<div class="ts-entry__outer">
|
||
<div class="ts-entry__inner ts-entry__thumbnail">
|
||
<div class="ts-overlay-background filter-mayfair">
|
||
<img src="https://slack.engineering/wp-content/uploads/sites/7/2025/05/deploy-safety-smaller.jpeg?w=380&h=250&crop=1" class="attachment-tinyspeck-thumbnail size-tinyspeck-thumbnail wp-post-image" alt="A rocket streaking across the sky wearing a life vest to symbolize deploying safely" decoding="async" loading="lazy" /> </div>
|
||
<a href="https://slack.engineering/deploy-safety/" class="ts-overlay-link"></a>
|
||
</div>
|
||
<div class="ts-entry__inner ts-entry__content">
|
||
<div class="ts-entry__post-meta">
|
||
<div class="ts-meta-date">October 7, 2025</div> <div class="ts-meta-reading-time">11 min read</div> </div>
|
||
<h2 class="ts-entry__title"><a href="https://slack.engineering/deploy-safety/"><span>Deploy Safety: Reducing customer impact from change</span></a></h2><span class="ts-mention">@Sam Bailey</span> </div>
|
||
</div>
|
||
</article>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<!-- Arc -->
|
||
<div class="c-arc concave-up">
|
||
<div class="c-arc__upper color-blue"></div>
|
||
<div class="c-arc__lower color-white"></div>
|
||
</div>
|
||
</div>
|
||
</main>
|
||
<footer class="ts-footer">
|
||
<div class="ts-container">
|
||
<div class="ts-footer__item">
|
||
<div class="ts-footer__col ts-col-left">
|
||
<div class="ts-footer__inner">
|
||
<div class="ts-logo">
|
||
<a class="ts-footer__logo" href="https://slack.engineering/">
|
||
<img src="https://slack.engineering/wp-content/themes/tinyspeck/assets/public/images/logo.svg" alt="Slack Engineering"/>
|
||
</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="ts-footer__col ts-col-canter">
|
||
<div class="ts-footer__inner">
|
||
<nav class="ts-footer__nav">
|
||
<div class="ts-footer__nav-item">
|
||
<ul class="ts-footer__nav-inner">
|
||
<li><a href="https://slack.engineering/articles/">Articles</a></li>
|
||
<!--<li><a href="https://slack.engineering/about/">About</a></li>-->
|
||
<li><a href="https://slack.com/careers/dept/software-engineering" target="_blank">Careers</a></li>
|
||
<li><a href="https://slack.dev/blog/" target="_blank">Slack Developer Blog</a></li>
|
||
<li><a href="https://slack.com/blog/" target="_blank">The Slack Blog</a></li>
|
||
</ul>
|
||
</div>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
<div class="ts-footer__col ts-col-right">
|
||
<div class="ts-footer__inner">
|
||
<span>
|
||
<a href="https://x.com/SlackEng" target="_blank" class="ts-share-buttons-twitter">
|
||
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M9.70487 6.85148L15.289 0.5H13.9657L9.11705 6.0149L5.24443 0.5H0.777832L6.63398 8.8395L0.777832 15.5H2.10116L7.22147 9.67608L11.3112 15.5H15.7778L9.70455 6.85148H9.70487ZM7.89239 8.91297L7.29905 8.08255L2.57797 1.47476H4.61052L8.42048 6.80746L9.01383 7.63788L13.9663 14.5696H11.9338L7.89239 8.91329V8.91297Z" fill="currentcolor"></path>
|
||
</svg>
|
||
</a>
|
||
<a href="https://www.youtube.com/@Slackhq" target="_blank" class="ts-share-buttons-youtube">
|
||
<svg width="21" height="14" viewBox="0 0 21 14" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path fill-rule="evenodd" clip-rule="evenodd" d="M10.463 0C19.0721 0 20.1483 0 20.1483 7C20.1483 14 19.0721 14 10.463 14C1.85397 14 0.777832 14 0.777832 7C0.777832 0 1.85397 0 10.463 0ZM8.46143 2.7998C8.10631 2.7998 7.77271 3.0798 7.77271 3.4998V10.4998C7.77271 10.9521 8.11707 11.1998 8.46143 11.1998C8.60133 11.1998 8.7197 11.1567 8.82732 11.0921L14.3694 7.59211C14.5846 7.47365 14.6923 7.26904 14.6923 6.9998C14.6923 6.73057 14.5846 6.52596 14.3694 6.4075L8.82732 2.9075C8.70894 2.83211 8.59057 2.7998 8.46143 2.7998Z" fill="currentcolor"></path>
|
||
</svg>
|
||
</a>
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<br/><br/>
|
||
<div class="ts-footer__item">
|
||
<p class="ts-footer__info">
|
||
<a href="https://slack.com/terms-of-service/user" target="_blank">Terms of Service</a>
|
||
<a href="https://slack.com/trust/privacy/privacy-policy" target="_blank">Privacy Information</a>
|
||
<a href="#" class="optanon-show-settings">Cookie Preferences</a>
|
||
<a href="https://www.salesforce.com/form/other/privacy-request/?_gl=1*1yvb56x*_gcl_au*OTY0OTg5MzcuMTcyNTkxMjE5Nw..*_ga*MTMzNjk3MzQ3MC4xNzE4MDQ0MzEy*_ga_QTJQME5M5D*MTcyNjI2OTEwNy40OS4xLjE3MjYyNjkyMzAuNS4wLjA." target="_blank">Your Privacy Choices</a>
|
||
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="30" height="14"><path d="M7.4 12.8h6.8l3.1-11.6H7.4C4.2 1.2 1.6 3.8 1.6 7s2.6 5.8 5.8 5.8z" style="fill-rule:evenodd;clip-rule:evenodd;fill:#fff"/><path d="M22.6 0H7.4c-3.9 0-7 3.1-7 7s3.1 7 7 7h15.2c3.9 0 7-3.1 7-7s-3.2-7-7-7zm-21 7c0-3.2 2.6-5.8 5.8-5.8h9.9l-3.1 11.6H7.4c-3.2 0-5.8-2.6-5.8-5.8z" style="fill-rule:evenodd;clip-rule:evenodd;fill:#06f"/><path d="M24.6 4c.2.2.2.6 0 .8L22.5 7l2.2 2.2c.2.2.2.6 0 .8-.2.2-.6.2-.8 0l-2.2-2.2-2.2 2.2c-.2.2-.6.2-.8 0-.2-.2-.2-.6 0-.8L20.8 7l-2.2-2.2c-.2-.2-.2-.6 0-.8.2-.2.6-.2.8 0l2.2 2.2L23.8 4c.2-.2.6-.2.8 0z" style="fill:#fff"/><path d="M12.7 4.1c.2.2.3.6.1.8L8.6 9.8c-.1.1-.2.2-.3.2-.2.1-.5.1-.7-.1L5.4 7.7c-.2-.2-.2-.6 0-.8.2-.2.6-.2.8 0L8 8.6l3.8-4.5c.2-.2.6-.2.9 0z" style="fill:#06f"/></svg>
|
||
<br/><br/>
|
||
© 2026 Slack Technologies, LLC, a Salesforce company. All rights reserved. Various trademarks held by their respective owners.
|
||
</p>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
|
||
<a href="#top" data-scroll-to-top class="ts-scroll-to-top">
|
||
<svg class="ts-icon-up" width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M5 12H19M19 12L12 5M19 12L12 19" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
|
||
<span>scroll to top</span>
|
||
</a>
|
||
|
||
<script type="speculationrules">
|
||
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/sites/7/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/tinyspeck/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
|
||
</script>
|
||
<script type="text/javascript" src="https://slack.engineering/_static/??-eJydj00KwjAQhS9kOxaK1YV4lpgM6cRkEjJTSm9vg7pxJS7fz/fgwVo6m1mRFUpcPLFAqSTJKFmIdH8pCG97L1fsgxzgdzAa9p39Byqbzpk/JLGNi0NpMSXjUWI2Dl2fiL/Xdca0N5V4k4L2AUYEVUCqbbiQthe3dB2m83Q6jpdhDE86bmNo" ></script><script id="jetpack-stats-js-before">
|
||
_stq = window._stq || [];
|
||
_stq.push([ "view", {"v":"ext","blog":"195077423","post":"17027","tz":"-7","srv":"slack.engineering","hp":"vip","j":"1:16.1.3"} ]);
|
||
_stq.push([ "clickTrackerInit", "195077423", "17027" ]);
|
||
//# sourceURL=jetpack-stats-js-before
|
||
</script>
|
||
<script data-wp-strategy="defer" defer fetchpriority="low" id="jetpack-stats-js" src="https://stats.wp.com/e-202637.js"></script>
|
||
<script id="wp-emoji-settings" type="application/json">
|
||
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://slack.engineering/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
|
||
</script>
|
||
<script type="module">
|
||
/*! This file is auto-generated */
|
||
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
|
||
//# sourceURL=https://slack.engineering/wp-includes/js/wp-emoji-loader.min.js
|
||
</script>
|
||
</body>
|
||
</html>
|