1343 lines
129 KiB
HTML
1343 lines
129 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en-US">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||
<meta name="mobile-web-app-capable" content="yes">
|
||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||
<meta name="apple-mobile-web-app-title" content="Engineering at Meta - Engineering at Meta Blog">
|
||
<meta property="fb:app_id" content="1425766027653270" />
|
||
<link rel="profile" href="http://gmpg.org/xfn/11">
|
||
<link rel="pingback" href="https://engineering.fb.com/xmlrpc.php">
|
||
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
|
||
<script async src="https://engineering.fb.com/wp-content/themes/code-fb-com/dist/js/optimize.min.js"></script>
|
||
|
||
<!-- This site is optimized with the Yoast SEO Premium plugin v19.3 (Yoast SEO v27.3) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
|
||
<title>Scaling services with Shard Manager - Engineering at Meta</title>
|
||
<meta name="description" content="Shard Manager provides an end-to-end solution supporting basic shard failover, load balancing, shard scaling, and operational safety." />
|
||
<link rel="canonical" href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/" />
|
||
<meta name="twitter:label1" content="Written by" />
|
||
<meta name="twitter:data1" content="Gerald Guo, Thawan Kooburat" />
|
||
<meta name="twitter:label2" content="Est. reading time" />
|
||
<meta name="twitter:data2" content="19 minutes" />
|
||
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/#article","isPartOf":{"@id":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/"},"author":[{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/e8e88f847c0823d1b37ca404058e2cab"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/6e8fe963d61fe822bf9e7f2fbc8cd108"}],"headline":"Scaling services with Shard Manager","datePublished":"2020-08-24T16:00:58+00:00","dateModified":"2023-10-17T19:51:29+00:00","mainEntityOfPage":{"@id":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/"},"wordCount":3726,"publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"image":{"@id":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2020\/08\/SHARD_manager_hero_01.jpg","articleSection":["Core Infra","Production Engineering"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/","url":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/","name":"Scaling services with Shard Manager - Engineering at Meta","isPartOf":{"@id":"https:\/\/engineering.fb.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/#primaryimage"},"image":{"@id":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2020\/08\/SHARD_manager_hero_01.jpg","datePublished":"2020-08-24T16:00:58+00:00","dateModified":"2023-10-17T19:51:29+00:00","description":"Shard Manager provides an end-to-end solution supporting basic shard failover, load balancing, shard scaling, and operational safety.","breadcrumb":{"@id":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/#primaryimage","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2020\/08\/SHARD_manager_hero_01.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2020\/08\/SHARD_manager_hero_01.jpg","width":1920,"height":1080},{"@type":"BreadcrumbList","@id":"https:\/\/engineering.fb.com\/2020\/08\/24\/production-engineering\/scaling-services-with-shard-manager\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/engineering.fb.com\/"},{"@type":"ListItem","position":2,"name":"Scaling services with Shard Manager"}]},{"@type":"WebSite","@id":"https:\/\/engineering.fb.com\/#website","url":"https:\/\/engineering.fb.com\/","name":"Engineering at Meta","description":"Engineering at Meta Blog","publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/engineering.fb.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/engineering.fb.com\/#organization","name":"Meta","url":"https:\/\/engineering.fb.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","width":29011,"height":12501,"caption":"Meta"},"image":{"@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/Engineering\/","https:\/\/x.com\/fb_engineering"]},[],{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/e8e88f847c0823d1b37ca404058e2cab","name":"Gerald Guo","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Gerald Guo"},"url":"https:\/\/engineering.fb.com\/author\/gerald-guo\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/6e8fe963d61fe822bf9e7f2fbc8cd108","name":"Thawan Kooburat","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Thawan Kooburat"},"url":"https:\/\/engineering.fb.com\/author\/thawan-kooburat\/"}]}</script>
|
||
<!-- / Yoast SEO Premium plugin. -->
|
||
|
||
|
||
<link rel='dns-prefetch' href='//secure.gravatar.com' />
|
||
<link rel='dns-prefetch' href='//v0.wordpress.com' />
|
||
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta » Feed" href="https://engineering.fb.com/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta » Comments Feed" href="https://engineering.fb.com/comments/feed/" />
|
||
<script async src="https://www.googleoptimize.com/optimize.js?id=OPT-TPNDQC5" onerror="dataLayer.hide.end && dataLayer.hide.end()"></script>
|
||
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2020%2F08%2F24%2Fproduction-engineering%2Fscaling-services-with-shard-manager%2F" />
|
||
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2020%2F08%2F24%2Fproduction-engineering%2Fscaling-services-with-shard-manager%2F&format=xml" />
|
||
<style id="wp-img-auto-sizes-contain-inline-css">
|
||
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
|
||
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
|
||
</style>
|
||
<style id="wp-emoji-styles-inline-css">
|
||
|
||
img.wp-smiley, img.emoji {
|
||
display: inline !important;
|
||
border: none !important;
|
||
box-shadow: none !important;
|
||
height: 1em !important;
|
||
width: 1em !important;
|
||
margin: 0 0.07em !important;
|
||
vertical-align: -0.1em !important;
|
||
background: none !important;
|
||
padding: 0 !important;
|
||
}
|
||
/*# sourceURL=wp-emoji-styles-inline-css */
|
||
</style>
|
||
<style id="wp-block-library-inline-css">
|
||
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
|
||
|
||
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
|
||
</style>
|
||
<style id="classic-theme-styles-inline-css">
|
||
/*! This file is auto-generated */
|
||
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
|
||
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
|
||
</style>
|
||
|
||
<style id="global-styles-inline-css">
|
||
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
|
||
/*# sourceURL=global-styles-inline-css */
|
||
</style>
|
||
|
||
<link rel='stylesheet' id='all-css-12' href='https://engineering.fb.com/_static/??-eJyNjUsOwjAMRC9EaiqghQXiKFVqhzSQn+pEiNuTFJBAbNjZTzNv4BYFBp+UTxBt1sYzFKQpzpVz5WQ4QSUv0CDzCj6KaVJOMWAgJc5jwe7ZKTnAyVgSS6Jxxv90XRbv3YtKUeJVtF3TgguUbZHyJGdFkui+nMbrvxWD8QhjLvvAAY20wgYd+OupspM7tv3+sO533Wb7AImQY4U=' type='text/css' media='all' />
|
||
<script id="gdprconsent-js" src="https://engineering.fb.com/wp-content/plugins/wp-gdpr-consent/dist/gdprconsent.js?ver=2.0.0"></script>
|
||
<script id="gdprconsent-js-after">
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack('https://www.googletagmanager.com/gtag/js?id=391063774');
|
||
window.gdprSafeTrack(function() {
|
||
var google_analytics_g4 = decodeURIComponent( '391063774' );
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
gtag('js', new Date());
|
||
gtag('config', google_analytics_g4);
|
||
});
|
||
}
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
var google_tag_manager = decodeURIComponent( 'GTM-KZRLGZ8' );
|
||
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||
})(window,document,'script','dataLayer',google_tag_manager);
|
||
});
|
||
}
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
var facebook_pixel = decodeURIComponent( '660742068067366' );
|
||
!function(f,b,e,v,n,t,s)
|
||
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
|
||
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
|
||
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
|
||
n.queue=[];t=b.createElement(e);t.async=!0;
|
||
t.src=v;s=b.getElementsByTagName(e)[0];
|
||
s.parentNode.insertBefore(t,s)}(window, document,'script',
|
||
'https://connect.facebook.net/en_US/fbevents.js');
|
||
fbq('init', facebook_pixel);
|
||
fbq('track', 'PageView');
|
||
});
|
||
}
|
||
//# sourceURL=gdprconsent-js-after
|
||
</script>
|
||
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJzTLy/QzcxLzilNSS3WzwKiwtLUokoopZebmaeXVayjj0+Rbm5melFiSSpUsX2uraG5hbmZgYmloUkWAK+3Iic=" ></script><script type="text/javascript" src="https://engineering.fb.com/wp-content/themes/understrap/js/popper.min.js?m=1773259670g" ></script><link rel="https://api.w.org/" href="https://engineering.fb.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.fb.com/wp-json/wp/v2/posts/16558" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.fb.com/xmlrpc.php?rsd" />
|
||
<meta name="generator" content="WordPress 7.1" />
|
||
<link rel='shortlink' href='https://wp.me/pa0Lhq-4j4' />
|
||
<link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/Optimistic_Display_W_Md.woff2" as="font" type="font/woff2" crossorigin="anonymous">
|
||
<!-- Jetpack Open Graph Tags -->
|
||
<meta property="og:type" content="article" />
|
||
<meta property="og:title" content="Scaling services with Shard Manager" />
|
||
<meta property="og:url" content="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/" />
|
||
<meta property="og:description" content="Over the years, as we’ve expanded in scale and functionalities, Facebook has evolved from a basic web server architecture into a complex one with thousands of services working behind the scenes. It…" />
|
||
<meta property="article:published_time" content="2020-08-24T16:00:58+00:00" />
|
||
<meta property="article:modified_time" content="2023-10-17T19:51:29+00:00" />
|
||
<meta property="og:site_name" content="Engineering at Meta" />
|
||
<meta property="og:image" content="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager_hero_01.jpg" />
|
||
<meta property="og:image:width" content="1920" />
|
||
<meta property="og:image:height" content="1080" />
|
||
<meta property="og:image:alt" content="" />
|
||
<meta property="og:locale" content="en_US" />
|
||
<meta name="twitter:text:title" content="Scaling services with Shard Manager" />
|
||
<meta name="twitter:image" content="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager_hero_01.jpg?w=640" />
|
||
<meta name="twitter:card" content="summary_large_image" />
|
||
|
||
<!-- End Jetpack Open Graph Tags -->
|
||
<style id="wp-custom-css">
|
||
.table-of-contents {
|
||
position: sticky;
|
||
top: 20px;
|
||
background: #fff;
|
||
padding: 1em;
|
||
border: 1px solid #ddd;
|
||
border-radius: 8px;
|
||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||
max-width: 250px;
|
||
}
|
||
</style>
|
||
|
||
<link rel="shortcut icon" href="https://engineering.fb.com/wp-content/themes/code-fb-com/favicon.ico" />
|
||
</head>
|
||
|
||
<body class="wp-singular post-template-default single single-post postid-16558 single-format-standard wp-theme-understrap wp-child-theme-code-fb-com group-blog">
|
||
|
||
<div class="hfeed site" id="page">
|
||
|
||
<!-- ******************* The Navbar Area ******************* -->
|
||
<div class="wrapper-fluid wrapper-navbar" id="wrapper-navbar">
|
||
<a class="skip-link screen-reader-text sr-only" href="#content">
|
||
Skip to content </a>
|
||
|
||
<nav class="navbar navbar-expand-std">
|
||
|
||
<div class="nav-container">
|
||
|
||
|
||
<!-- Your site title as branding in the menu -->
|
||
|
||
<div class="nav-top-row">
|
||
|
||
<h1 class="navbar-brand mb-0">
|
||
<a rel="home" href="https://engineering.fb.com/" title="Engineering at Meta">
|
||
<img class="navbar-brand-logo" src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/logo-meta.svg" class="navbar-brand-logo" alt="Engineering at Meta" width="215" height="40" loading="lazy">
|
||
</a>
|
||
</h1>
|
||
|
||
<!-- end custom logo -->
|
||
<label for="search" class="screen-reader-text">Search this site</label>
|
||
<input type="text" id="search" name="search" class="form-control-lg navbar-search" autocomplete="off">
|
||
|
||
<button class="navbar-toggler navbar-searchbar-toggler"
|
||
type="button"
|
||
data-toggle="collapse"
|
||
data-target="#searchbarDropdown"
|
||
aria-controls="searchbarDropdown"
|
||
aria-expanded="false"
|
||
aria-label="Toggle navigation">
|
||
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/icon-search.svg" class="navbar-mobile-search-icon" loading="lazy" width="24" height="25" />
|
||
</button>
|
||
|
||
<!-- desktop language menu (mobile language menu is implemented via jQuery)-->
|
||
<div class="notranslate nice-select navbar-language form-control">
|
||
</div>
|
||
|
||
<button class="navbar-toggler collapsed navbar-menu-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
|
||
</div>
|
||
|
||
<div class="navbar-border"></div>
|
||
|
||
<!-- The WordPress Menu goes here -->
|
||
<div id="navbarNavDropdown" class="collapse navbar-collapse"><ul id="main-menu" class="navbar-nav"><li id="menu-item-60" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-60 dropdown"><a title="Open Source" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Open Source <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-12330" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-12330"><a title="Open Source" href="https://engineering.fb.com/category/open-source/" class="nav-link">Open Source</a></li>
|
||
<li id="menu-item-12331" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-12331"><a title="Meta Open Source" href="https://opensource.fb.com" class="nav-link">Meta Open Source</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2686" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2686 dropdown"><a title="Platforms" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Platforms <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-65" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-65"><a title="Android" href="https://engineering.fb.com/category/android/" class="nav-link">Android</a></li>
|
||
<li id="menu-item-66" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-66"><a title="iOS" href="https://engineering.fb.com/category/ios/" class="nav-link">iOS</a></li>
|
||
<li id="menu-item-67" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-67"><a title="Web" href="https://engineering.fb.com/category/web/" class="nav-link">Web</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2682" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2682 dropdown"><a title="Infrastructure Systems" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Infrastructure Systems <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-218" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-218"><a title="Core Infra" href="https://engineering.fb.com/category/core-infra/" class="nav-link">Core Infra</a></li>
|
||
<li id="menu-item-219" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-219"><a title="Data Infrastructure" href="https://engineering.fb.com/category/data-infrastructure/" class="nav-link">Data Infrastructure</a></li>
|
||
<li id="menu-item-220" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-220"><a title="DevInfra" href="https://engineering.fb.com/category/developer-tools/" class="nav-link">DevInfra</a></li>
|
||
<li id="menu-item-226" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-226"><a title="Production Engineering" href="https://engineering.fb.com/category/production-engineering/" class="nav-link">Production Engineering</a></li>
|
||
<li id="menu-item-227" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-227"><a title="Security & Privacy" href="https://engineering.fb.com/category/security/" class="nav-link">Security & Privacy</a></li>
|
||
<li id="menu-item-18607" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18607"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/systems-infrastructure/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2683" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2683 dropdown"><a title="Physical Infrastructure" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Physical Infrastructure <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-215" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-215"><a title="Connectivity" href="https://engineering.fb.com/category/connectivity/" class="nav-link">Connectivity</a></li>
|
||
<li id="menu-item-216" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-216"><a title="Data Center Engineering" href="https://engineering.fb.com/category/data-center-engineering/" class="nav-link">Data Center Engineering</a></li>
|
||
<li id="menu-item-217" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-217"><a title="Networking & Traffic" href="https://engineering.fb.com/category/networking-traffic/" class="nav-link">Networking & Traffic</a></li>
|
||
<li id="menu-item-18608" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18608"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/networking-connectivity/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2684" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2684 dropdown"><a title="Video Engineering & AR/VR" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Video Engineering & AR/VR <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-225" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-225"><a title="Video Engineering" href="https://engineering.fb.com/category/video-engineering/" class="nav-link">Video Engineering</a></li>
|
||
<li id="menu-item-223" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-223"><a title="Virtual Reality" href="https://engineering.fb.com/category/virtual-reality/" class="nav-link">Virtual Reality</a></li>
|
||
<li id="menu-item-13388" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13388"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/augmented-reality-virtual-reality/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2685" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2685 dropdown"><a title="Artificial Intelligence" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Artificial Intelligence <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-222" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-222"><a title="ML Applications" href="https://engineering.fb.com/category/ml-applications/" class="nav-link">ML Applications</a></li>
|
||
<li id="menu-item-221" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-221"><a title="AI Research" href="https://engineering.fb.com/category/ai-research/" class="nav-link">AI Research</a></li>
|
||
<li id="menu-item-13387" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13387"><a title="Research Publications" href="https://ai.facebook.com/results/?content_types%5B0%5D=publication" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-228" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-228"><a title="Watch Videos" href="/videos" class="nav-link">Watch Videos</a></li>
|
||
</ul></div> </div><!-- .container -->
|
||
|
||
<div class="search-container collapse" id="searchbarDropdown">
|
||
<div class="searchbar-search">
|
||
<div class="searchbar-container">
|
||
|
||
<div class="navbar-mobile-search-margin"></div>
|
||
<input type="text" name="search" placeholder="Search" class="form-control-lg navbar-mobile-search" autofocus autocomplete="off">
|
||
<button class="searchbar-clear">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</nav><!-- .site-navigation -->
|
||
|
||
</div><!-- .wrapper-navbar end -->
|
||
|
||
<div class="wrapper" id="single-wrapper">
|
||
|
||
<div class="single-container row" id="content" tabindex="-1">
|
||
|
||
<!-- Do the left sidebar check. -->
|
||
|
||
|
||
|
||
<div class="content-area" id="primary">
|
||
<main class="site-main" id="main">
|
||
|
||
|
||
|
||
<article class="post-16558 post type-post status-publish format-standard has-post-thumbnail hentry category-core-infra category-production-engineering fb_content_type-article" id="post-16558">
|
||
|
||
<header class="entry-header">
|
||
<div class="entry-meta">
|
||
POSTED ON <span class="posted-on entry-date"><time class="published updated" datetime="2020-08-24">AUGUST 24, 2020</time></span> TO <span class="cat-links"><a href="https://engineering.fb.com/category/core-infra/" rel="category tag">Core Infra</a>, <a href="https://engineering.fb.com/category/production-engineering/" rel="category tag">Production Engineering</a></span>
|
||
<h1 class="entry-title"><div>Scaling services with Shard Manager</div></h1> </div><!-- .entry-meta -->
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
<figure id="post-feat-image-container">
|
||
<img width="1920" height="1080" src="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager_hero_01.jpg" class="wp-post-image" alt="" title="SHARD_manager_hero_01" /> </figure>
|
||
|
||
<div class="entry-authors-social">
|
||
<div class="entry-authors">
|
||
By <a href="https://engineering.fb.com/author/gerald-guo/" title="Posts by Gerald Guo" class="author url fn" rel="author">Gerald Guo</a>, <a href="https://engineering.fb.com/author/thawan-kooburat/" title="Posts by Thawan Kooburat" class="author url fn" rel="author">Thawan Kooburat</a> </div>
|
||
</div>
|
||
|
||
|
||
<div class="entry-content">
|
||
|
||
<p><span style="font-weight: 400;">Over the years, as we’ve expanded in scale and functionalities, Facebook has evolved from a basic web server architecture into a complex one with thousands of services working behind the scenes. It’s no trivial task to scale the wide range of back-end services needed for Facebook’s products. And we found that many of our teams were building their own custom sharding solutions with overlapping functionalities. To address this problem, we built Shard Manager as a generic platform that facilitates efficient development and operation of reliable sharded applications.</span></p>
|
||
<p><span style="font-weight: 400;">The concept of using sharding to scale services is not new. However, to the best of our knowledge, we are the only generic sharding platform in the industry that achieves wide adoption at our scale. Shard Manager manages tens of millions of shards hosted on hundreds of thousands of servers across hundreds of applications in production.</span></p>
|
||
<h2><span style="font-weight: 400;">An introduction to sharding</span></h2>
|
||
<p><span style="font-weight: 400;">In the most basic form, people are familiar with sharding as a way to scale out services to support high throughput. The figure below illustrates the scaling of a typical web stack. The web tier is usually stateless and easy to scale out. As any server can handle any request, a wide range of traffic routing strategies can be used, like round robin or random.</span></p>
|
||
<figure id="attachment_16565" aria-describedby="caption-attachment-16565" style="width: 916px" class="wp-caption alignnone"><img decoding="async" class="wp-image-16565 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-01-e1599251955247.jpg?w=916" alt="" width="916" height="373" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-01-e1599251955247.jpg 1920w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-01-e1599251955247.jpg?resize=916,373 916w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-01-e1599251955247.jpg?resize=768,313 768w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-01-e1599251955247.jpg?resize=1024,417 1024w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-01-e1599251955247.jpg?resize=1536,626 1536w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-01-e1599251955247.jpg?resize=96,39 96w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-01-e1599251955247.jpg?resize=192,78 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-16565" class="wp-caption-text">The Facebook application stack.</figcaption></figure>
|
||
<p><span style="font-weight: 400;">On the other hand, scaling the database part is nontrivial due to its state. We need to apply a scheme to deterministically spread data across servers. A simple scheme of hashing like </span><code class="language-none">hash(data_key) % num_servers</code> <span style="font-weight: 400;">can spread data but has the problem of shuffling data around when servers are added to scale. Consistent hashing addresses the issue by redistributing only a small subset of data from existing servers to new servers. However, this scheme requires application to have fine-grained keys for statistical load balancing to be effective. Consistent hashing’s ability to support constraint-based allocation (e.g., data of European Union users should be stored in European data centers for lower latency) is also limited due to its nature. As a result, only certain classes of applications such as distributed caching adopt this scheme.</span></p>
|
||
<p><span style="font-weight: 400;">An alternative scheme is to explicitly partition data into shards that are allocated to servers. The data of billions of users is stored across many database instances, and each instance can be treated as a shard. To increase fault tolerance, each database shard can have multiple copies (aka replicas), each of which can play different roles</span> <span style="font-weight: 400;">(e.g., primary or secondary) depending on the consistency requirement.</span></p>
|
||
<p><span style="font-weight: 400;">The allocation of shards to servers is explicitly computed with the capability of incorporating various constraints like locality preference, which hashing solutions cannot support. We found that the sharding approach is more flexible than hashing and suits the needs of a wider range of distributed applications.</span></p>
|
||
<p><span style="font-weight: 400;">Applications adopting this sharding approach often need certain shard management capabilities in order to operate reliably at scale. The most basic one is the ability to failover. In the event of a hardware or software failure, the system can divert client traffic away from failed servers and may even need to rebuild impacted replicas on healthy servers. In large data centers, there is always planned downtime for servers to perform hardware or software maintenance</span><i><span style="font-weight: 400;">. </span></i><span style="font-weight: 400;">The shard management system needs to ensure that each shard has enough healthy replicas by proactively moving replicas away from servers to be taken down if deemed necessary.</span></p>
|
||
<p><span style="font-weight: 400;">Additionally, the possibly uneven and ever-changing shard load requires load balancing, meaning the set of shards each server hosts must be dynamically adjusted to achieve uniform resource utilization and improve overall resource efficiency and service reliability. Finally, the fluctuation of client traffic requires shard-scaling, where the system dynamically adjusts replication factor on a per-shard basis to ensure its average per-replica load stays optimal.</span></p>
|
||
<p><span style="font-weight: 400;">We discovered that different service teams at Facebook were already building their own custom solutions to varying degrees of completeness. It was common to see services able to handle failover but have a very limited form of load balancing. This led to suboptimal reliability and high operation overhead. This is why we designed Shard Manager as a generic shard management platform.<br />
|
||
</span></p>
|
||
<div class="fb-video" data-allowfullscreen="true" data-href="https://www.facebook.com/1576488989290866/videos/606214686732164" style="background-color: #fff; display: inline-block;"></div>
|
||
<p> </p>
|
||
<h2><span style="font-weight: 400;">Sharding as a platform with Shard Manager</span></h2>
|
||
<p><span style="font-weight: 400;">Over the years, hundreds of sharded applications have been built or migrated onto Shard Manager, totaling upper tens of millions of shard replicas on upper hundreds of thousands of servers with historical hypergrowth, as shown in the figure below.</span></p>
|
||
<p><span style="font-weight: 400;">These applications assist with the smooth functioning of various user-facing products, including the Facebook app, Messenger, WhatsApp, and Instagram.</span></p>
|
||
<figure id="attachment_16566" aria-describedby="caption-attachment-16566" style="width: 916px" class="wp-caption alignnone"><img decoding="async" class="wp-image-16566 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-02.jpg?w=916" alt="" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-02.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-02.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-02.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-02.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-02.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-02.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-02.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-02.jpg?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-16566" class="wp-caption-text">The growth of the total number of application servers.</figcaption></figure>
|
||
<p><span style="font-weight: 400;">Apart from the sheer number of applications, their use cases vary significantly in both complexity and scale, ranging from simple counter service with high tens of servers to complex, Paxos-based global storage service with tens of thousands of servers. The image below shows a wide spectrum of representative applications, with their scale indicated by font size.</span></p>
|
||
<figure id="attachment_16567" aria-describedby="caption-attachment-16567" style="width: 916px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="wp-image-16567 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-03.jpg?w=916" alt="" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-03.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-03.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-03.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-03.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-03.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-03.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-03.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-03.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-16567" class="wp-caption-text">Representative applications on Shard Manager.</figcaption></figure>
|
||
<p><span style="font-weight: 400;">Various factors contribute to the wide adoption. First, integrating with Shard Manager means simply implementing a small, straightforward interface consisting of </span><code class="language-none">add_shard </code><span style="font-weight: 400;">and </span><code class="language-none">drop_shard </code><span style="font-weight: 400;">primitives. Second, each application can declare its reliability and efficiency requirements via intent-based specification. Third, the use of a generic constrained optimization solver enables Shard Manager to provide versatile load-balancing capability and easily add support for new balancing strategies.</span></p>
|
||
<p><span style="font-weight: 400;">Last but not least, by fully integrating into the overall infrastructure ecosystem, including capacity and container management, Shard Manager supports not only efficient development but also safe operation of sharded applications, and therefore provides an end-to-end solution, which no similar platforms provide. Shard Manager supports more sophisticated use cases than do similar platforms, like</span><a href="https://helix.apache.org/"> <span style="font-weight: 400;">Apache Helix</span></a><span style="font-weight: 400;">, including Paxos-based storage system use cases.</span></p>
|
||
<h2><span style="font-weight: 400;">Types of Shard Manager applications</span></h2>
|
||
<p><span style="font-weight: 400;">We abstract the commonality of applications on Shard Manager and categorize them into three types: primary only, secondaries only, and primary-secondaries.</span></p>
|
||
<p><b>Primary only:<br />
|
||
</b><span style="font-weight: 400;">Each shard has a single replica, called the primary replica. These types of applications typically store state in external systems, like databases and data warehouses. A common paradigm is that each shard represents a worker that fetches designated data, processes them, optionally serves client requests, and writes back the results, with optional optimizations like batching. Stream processing is one real example that processes data from an input stream and writes results to an output stream. Shard Manager provides an at-most-one-primary guarantee to help prevent data inconsistency due to duplicate data processing, like a traditional ZooKeeper lock-based approach.</span></p>
|
||
<p><b>Secondaries only:<br />
|
||
</b><span style="font-weight: 400;">Each shard has multiple replicas of equal role, dubbed secondaries. The redundancy from multiple replicas provides better fault tolerance. Additionally, replication factor can be adjusted based on workload: Hot shards can have more replicas to spread load. Typically, these types of applications are read-only without strong consistency requirements. They fetch data from an external storage system, optionally process the data, cache results locally, and serve queries off local data. One real example is machine learning inference systems, which download trained models from remote storage and serve inference requests.</span></p>
|
||
<p><b>Primary-secondaries:<br />
|
||
</b><span style="font-weight: 400;">Each shard has multiple replicas of two roles — primary and secondary. These types of applications are typically storage systems with strict requirements on data consistency and durability, where the primary replica accepts write requests and drives replication among all replicas while secondaries provide redundancy and can optionally serve reads to reduce the load on the primary. One example is</span> <a href="https://engineering.fb.com/2021/08/06/core-data/zippydb/">Zippy DB</a><span style="font-weight: 400;">, which is a global key-value store with Paxos-based replication.</span></p>
|
||
<p><span style="font-weight: 400;">We found that the above three types can model most sharded applications at Facebook, with percentage distribution as of August 2020 shown below. Sixty-seven percent of applications are primary-only due to architectural simplicity and conceptual similarity to traditional ZooKeeper’s lock-based solutions. However, in terms of server count, primary-only accounts for only 17 percent, which means primary-only applications are on average smaller than applications of the other two types.</span></p>
|
||
<figure id="attachment_16568" aria-describedby="caption-attachment-16568" style="width: 916px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="wp-image-16568 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-04.jpg?w=916" alt="" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-04.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-04.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-04.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-04.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-04.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-04.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-04.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-04.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-16568" class="wp-caption-text">Percentage distribution by application count and server count as of August 2020.</figcaption></figure>
|
||
<h2><span style="font-weight: 400;">Building applications with Shard Manager</span></h2>
|
||
<p><span style="font-weight: 400;">After application owners decide how to slice their workload/data into shards and which application type suits their needs, there are three straightforward, standardized steps in building a sharded application on Shard Manager, irrespective of use case.</span></p>
|
||
<ol>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">An application links a Shard Manager library and implements shard state transition interface with their business logic plugged in.</span></li>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">An application owner provides intent-based specification to configure constraints. Shard Manager offers four major groups of out-of-the-box functionalities: fault tolerance, load balancing, shard scaling, and operational safety.</span></li>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">Application clients use a common routing library to route shard-specific requests.</span></li>
|
||
</ol>
|
||
<p><b>Shard state transition interface<br />
|
||
</b><span style="font-weight: 400;">Our shard state transition interface consists of a small but solid set of imperative primitives</span> <span style="font-weight: 400;">shown below, via which application-specific logic is plugged: </span></p>
|
||
<pre class="line-numbers"><code class="language-none">status add_shard(shard_id)
|
||
status drop_shard(shard_id)</code></pre>
|
||
<p><span style="font-weight: 400;">The <code class="language-none">add_shard</code></span><span style="font-weight: 400;"> call instructs a server to load the shard identified by the passed-in shard ID. Return value indicates the status of the transition, such as whether shard loading is in progress or runs into errors. Conversely, the <code class="language-none">drop_shard</code></span> <span style="font-weight: 400;">call instructs a server to drop a shard and stop serving client requests.</span></p>
|
||
<p><span style="font-weight: 400;">This interface gives applications full freedom of mapping shards to their domain-specific data. For storage service, the <code class="language-none">add_shard</code></span><span style="font-weight: 400;"> call typically triggers data transfer from a peer replica; for a machine learning inference platform, the <code class="language-none">add_shard</code></span><span style="font-weight: 400;"> call triggers the loading of a model from remote storage to local host.</span></p>
|
||
<p><span style="font-weight: 400;">On top of the above primitives, Shard Manager constructs a high-level shard move protocol, illustrated below. Shard Manager decides to move a shard from server A, which is highly loaded, to the more lightly loaded server B to balance load. First, Shard Manager makes a <code class="language-none">drop_shard</code></span><span style="font-weight: 400;"> call to server A and waits for it to succeed. Second, it makes an <code class="language-none">add_shard</code></span><span style="font-weight: 400;"> call to server B. This protocol provides an at-most-one-primary guarantee.</span></p>
|
||
<figure id="attachment_16569" aria-describedby="caption-attachment-16569" style="width: 916px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="wp-image-16569 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-05.jpg?w=916" alt="" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-05.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-05.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-05.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-05.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-05.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-05.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-05.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-05.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-16569" class="wp-caption-text">Shard move across servers.</figcaption></figure>
|
||
<p><span style="font-weight: 400;">The above two basic primitives are all that a typical application needs to implement to become sharded and achieve scalability. For sophisticated applications, Shard Manager supports more powerful interfaces, which are elaborated on below.</span></p>
|
||
<p><span style="font-weight: 400;">In the above protocol, the clients of the shard under transition experiences transient unavailability during the period when the shard is not on any server, which is not acceptable for user-facing applications. As a result, we developed a more sophisticated protocol that supports seamless ownership handoff and minimizes shard downtime.</span></p>
|
||
<p><span style="font-weight: 400;">For primary-secondaries applications, two additional primitives are provided, as shown below</span><b>:</b></p>
|
||
<pre class="line-numbers"><code class="language-none">status change_role(shard_id, primary <-> secondary)
|
||
status update_membership(shard_id, [m1, m2, ...])</code></pre>
|
||
<ul>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;"><code class="language-none">change_role</code> </span><span style="font-weight: 400;">is used to transition the role of a replica between secondary and primary.</span></li>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;"><code class="language-none">update_membership</code> i</span><span style="font-weight: 400;">nstructs the primary of a shard to validate and execute replica membership change, which is important for Paxos-based applications to maximize data correctness.</span></li>
|
||
</ul>
|
||
<p><span style="font-weight: 400;">The above interfaces are the result of our thorough analysis and experience working with sharded applications. They turn out to be sufficiently generic to support most applications.</span></p>
|
||
<h2><span style="font-weight: 400;">Intent-based specification of various functionalities</span></h2>
|
||
<h3><span style="font-weight: 400;">Fault tolerance</span></h3>
|
||
<p><span style="font-weight: 400;">For distributed systems, failure is the norm rather than the exception, and knowing how to prepare for and recover from failure is essential to achieving high availability.</span></p>
|
||
<p><span style="font-weight: 400;"><strong>Replication:</strong> Redundancy via replication is a common strategy for improving fault tolerance. Shard Manager enables the configuration of replication factors on a per-shard basis. The benefit of replication is marginal if the failure of a single fault domain can take down all redundant replicas. Shard Manager supports spreading shard replicas across configurable fault domains, for instance, data center buildings for regional applications and regions for global applications.</span></p>
|
||
<p><span style="font-weight: 400;"><strong>Automatic failure detection and shard failover:</strong> Shard Manager can automatically detect server failures and network partition. After a failure is detected, it’s not always ideal to build replacement replicas immediately. Shard Manager enables applications to make an appropriate trade-off between the cost of building new replicas and acceptable unavailability via the configuration of failure detection latency and shard failover delay. Additionally, when network partitioning occurs, applications have the option of choosing between availability and consistency.</span></p>
|
||
<p><span style="font-weight: 400;"><strong>Failover throttling:</strong> To prevent cascading failures, Shard Manager supports failover throttling, which limits the rate of shard failover and protects the rest of the healthy servers from being suddenly overloaded in major outage scenarios.</span></p>
|
||
<h3><span style="font-weight: 400;">Load balancing</span></h3>
|
||
<p><span style="font-weight: 400;">Load balancing refers to the process of evenly distributing shards and workload thereof across application servers on a continual basis. It enables efficient use of resources and avoids hot spots.</span></p>
|
||
<p><span style="font-weight: 400;"><strong>Heterogeneous hardware and shards:</strong> At Facebook, we have multiple types and generations of hardware. Most applications need to run on heterogeneous hardware. The size and load of shards can vary for applications whose workload/data cannot be evenly sharded. Shard Manager load balancing algorithm takes fine-grained per-server and per-shard (replica) information into consideration and therefore supports both heterogeneous hardware and shards.</span></p>
|
||
<p><span style="font-weight: 400;"><strong>Dynamic load collection:</strong> A shard’s load can change over time due to its usage. The capacity available to applications can vary if it ties to a dynamic resource like available disk space. Shard Manager periodically collects per-replica load and per-server capacity from applications and instantiates load balancing.</span></p>
|
||
<p><span style="font-weight: 400;"><strong>Multi-resource balancing:</strong> Shard Manager supports simultaneously balancing multiple resources like compute, memory, and storage with varied user-configurable priorities. It makes sure the utilization of bottleneck resources falls within acceptable range, and that usage of less critical resources is balanced as much as possible.</span></p>
|
||
<p><span style="font-weight: 400;"><strong>Throttling:</strong> Similar to failover throttling, the number of shard moves generated by load balancing is throttled at the granularity of total moves and per-server moves.</span></p>
|
||
<p><span style="font-weight: 400;">The above versatile support for both spatial and temporal load variability suits the varying balancing needs of sharded applications.</span></p>
|
||
<h3><span style="font-weight: 400;">Shard scaling</span></h3>
|
||
<p><span style="font-weight: 400;">Lots of applications at Facebook serve requests from users directly or indirectly. Therefore, the traffic exhibits a diurnal pattern with a significant drop-off in request rate between peak and off-peak time.</span></p>
|
||
<p><span style="font-weight: 400;">Elastic compute, which dynamically adjusts resource allocation based on workload change, is one solution for improving resource efficiency without sacrificing reliability. In reaction to real-time load change, Shard Manager can perform shard scaling, which means it can dynamically adjust the replication factor when the average per-replica load of a shard deviates from a user-configured acceptable range. Shard scaling throttling can be configured to limit the number of added or dropped replicas in a given period.</span></p>
|
||
<p><span style="font-weight: 400;">The image below illustrates the scaling process for one shard. Initially, the total load across all replicas increases, and therefore per-replica load increases. Once per-replica load exceeds the upper threshold, shard scaling kicks in and adds a sufficient number of new replicas to bring the per-replica load back to an acceptable range. Later, the shard load starts to decrease, and shard scaling reduces the replica count to free unneeded resources to be used by other hot shards or applications.</span></p>
|
||
<figure id="attachment_16570" aria-describedby="caption-attachment-16570" style="width: 916px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="wp-image-16570 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-06.jpg?w=916" alt="" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-06.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-06.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-06.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-06.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-06.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-06.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-06.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-06.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-16570" class="wp-caption-text">An illustration of the shard scaling process.</figcaption></figure>
|
||
<h3><span style="font-weight: 400;">Operational safety</span></h3>
|
||
<p><span style="font-weight: 400;">Apart from failures, operational events are also the norm rather than exception and are treated as first-class citizens to minimize their impact on reliability. Common operational events include binary updates, hardware repair and maintenance, and kernel upgrades. Shard Manager is codesigned with</span><a href="https://engineering.fb.com/data-center-engineering/tupperware/"> <span style="font-weight: 400;">container management system Twine</span></a><span style="font-weight: 400;"> for seamless event handling. Twine aggregates the events, converts them to container life-cycle events like container stop/restart/move, and communicates them to Shard Manager Scheduler via the</span><a href="https://engineering.fb.com/data-center-engineering/tupperware/"> <span style="font-weight: 400;">TaskControl interface</span></a><span style="font-weight: 400;">.</span></p>
|
||
<p><span style="font-weight: 400;">Shard Manager Scheduler evaluates the disruptiveness and length of events, and makes necessary proactive shard moves to prevent the events from affecting reliability. Shard Manager guarantees the invariant that every shard must have at least one healthy replica. For Paxos-based applications with a majority quorum rule, Shard Manager supports a variant of the guarantee that ensures that the majority of replicas are healthy. Trade-offs between operational safety and efficiency vary by application and can be tuned via configuration, such as the upper limit of simultaneously affected shards.</span></p>
|
||
<p><span style="font-weight: 400;">The image below shows one example for an application with four containers and three shards. First, a short-lived maintenance operation like a kernel upgrade or security patch affecting Container 4 is requested, and Shard Manager allows the operation to immediately proceed because all shards still have sufficient replicas on the rest of servers. Next, a binary update is requested for Containers 1 to 3. Observing that updating any two containers concurrently would cause shard unavailability, Shard Manager applies the updates sequentially to the containers, one at a time.</span></p>
|
||
<figure id="attachment_16571" aria-describedby="caption-attachment-16571" style="width: 916px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="wp-image-16571 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-07.jpg?w=916" alt="" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-07.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-07.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-07.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-07.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-07.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-07.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-07.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-07.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-16571" class="wp-caption-text">An example of operational events handling.</figcaption></figure>
|
||
<h3><span style="font-weight: 400;">Client request routing</span></h3>
|
||
<p><span style="font-weight: 400;">We use a common routing library to route requests at Facebook. The routing library takes an application’s name and shard ID as input and returns a RPC client object via which RPC calls can be simply made, as illustrated in the following code. The magic of discovering where shards are allocated is hidden behind <code class="language-none">create_rpc_client</code></span><span style="font-weight: 400;">.</span></p>
|
||
<pre class="line-numbers"><code class="language-none">rpc_client = create_rpc_client(app_name, shard_id)
|
||
rpc_client.foo(...)</code></pre>
|
||
<h2><span style="font-weight: 400;">Shard Manager design and implementation</span></h2>
|
||
<p><span style="font-weight: 400;">In this section, we will dive into how Shard Manager is built to support the functionalities we discussed. We will start by sharing the layering of our infrastructure and especially the role of Shard Manager.</span></p>
|
||
<h3><span style="font-weight: 400;">Layering of the infrastructure stack</span></h3>
|
||
<p><span style="font-weight: 400;">At Facebook, our overall infrastructure has been built with a layered approach with a clear separation of concerns across layers. This enables us to evolve and scale each layer independently and robustly. The image below shows the layering of our infrastructure. Each layer allocates and defines the scope that the adjacent upper layer operates in. </span></p>
|
||
<figure id="attachment_16572" aria-describedby="caption-attachment-16572" style="width: 916px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="wp-image-16572 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-08.jpg?w=916" alt="" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-08.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-08.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-08.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-08.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-08.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-08.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-08.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-08.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-16572" class="wp-caption-text">The infrastructure stack.</figcaption></figure>
|
||
<ol>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">Host management: The Resource Allowance System manages all physical servers and allocates capacity to organizations and teams.</span></li>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">Container management: Twine gets capacity from the Resource Allowance System and allocates it to individual applications in the unit of containers.</span></li>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">Shard management: Shard Manager allocates shards within containers provided by Twine, for sharded applications.</span></li>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">Sharded applications: Within each shard, applications allocate and run the associated workload.</span></li>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">Products: These are user-facing products, like mobile apps, that are powered by the sharded back-end applications.</span></li>
|
||
</ol>
|
||
<p><span style="font-weight: 400;">Apart from the downward functional dependency of each layer on the adjacent lower layer, the whole infrastructure stack is codesigned and works collaboratively via signals and events propagating upward. For the Shard Manager layer specifically,</span><a href="https://engineering.fb.com/data-center-engineering/tupperware/"> <span style="font-weight: 400;">TaskControl</span></a><span style="font-weight: 400;"> is our mechanism of achieving collaborative scheduling.</span></p>
|
||
<h3><span style="font-weight: 400;">Designs</span></h3>
|
||
<p><b>Central control plane<br />
|
||
</b><span style="font-weight: 400;">Shard Manager is a pure control plane service that monitors application states and orchestrates moving application data across servers in shards. The centralized global view enables Shard Manager to compute globally optimal shard allocation and ensure high availability by holistically coordinating all planned operational events. In the event that this central control plane is down, applications can continue to function in degraded mode with existing shard allocation.</span></p>
|
||
<p><b>Opaque shards with state transition interface<br />
|
||
</b><span style="font-weight: 400;">Shards are opaque to Shard Manager, and users can map it to any entity in their application, such as database instances, groups of logs, and buckets of data. We define the shard state transition interfaces that each application must implement. This clean delineation sets Shard Manager apart from application-specific data planes and provides huge flexibility in terms of use cases that can leverage Shard Manager.</span></p>
|
||
<p><b>Shard granularity sweet spot<br />
|
||
</b><span style="font-weight: 400;">Shard granularity is important. Granularity that is too coarse results in poor load balancing, whereas granularity that is too fine incurs unnecessary management overhead on the underlying infrastructure. Our deliberately chosen sweet spot of allocating hundreds of shards to each application server strikes a good balance between load-balancing quality and infrastructure cost.</span></p>
|
||
<p><b>Generic constrained optimization<br />
|
||
</b><span style="font-weight: 400;">One manifestation of use case diversity is the assorted ways in which applications want shards to be allocated to achieve fault tolerance and efficiency. We adopt a generic</span><a href="https://en.wikipedia.org/wiki/Constrained_optimization"> <span style="font-weight: 400;">constrained optimization</span></a><span style="font-weight: 400;"> solver to achieve extensibility. When adding support for new requirements, Shard Manager just needs to internally formulate it as constraints and feed them into the solver to compute optimal shard allocation, with little complexity added to our codebase.</span></p>
|
||
<h3><span style="font-weight: 400;">Architecture</span></h3>
|
||
<p><span style="font-weight: 400;">This shows the architecture of Shard Manager, with various components explained below.</span></p>
|
||
<figure id="attachment_16573" aria-describedby="caption-attachment-16573" style="width: 916px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="wp-image-16573 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-09.jpg?w=916" alt="" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-09.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-09.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-09.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-09.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-09.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-09.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-09.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/08/SHARD_manager-09.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-16573" class="wp-caption-text">The architecture of Shard Manager.</figcaption></figure>
|
||
<p><span style="font-weight: 400;">A specification, with all the information needed to manage an application, is provided by application owners to Shard Manager Scheduler.</span></p>
|
||
<p><span style="font-weight: 400;">Shard Manager Scheduler is the central service that orchestrates shard transitions and moves. It collects application states; monitors state changes like server join, server failure, and load change; adjusts shard allocation; and drives shard state transitions via RPC calls to application servers. Shard Manager Scheduler is internally sharded to scale out horizontally. </span></p>
|
||
<p><span style="font-weight: 400;">Applications</span> <span style="font-weight: 400;">link the Shard Manager library, which provides server membership and liveness check transparency via connection to ZooKeeper. Applications implement the shard state transition interface and are instructed by Shard Manager Scheduler for state transitions. Applications can measure and expose dynamic load information, which is collected by Shard Manager Scheduler.</span></p>
|
||
<p><span style="font-weight: 400;">Shard Manager Scheduler publishes a public view of the shard allocation to a highly available and scalable service discovery system, which propagates the information to application clients for routing of requests.</span></p>
|
||
<p><span style="font-weight: 400;">Application clients link a common routing library, which encapsulates the discovery of server endpoints on a per-shard basis. After endpoint discovery, client requests are directly sent to the application servers. Therefore, shard manager scheduler is not on the critical path of request serving.</span></p>
|
||
<h2><span style="font-weight: 400;">Conclusions and upcoming challenges</span></h2>
|
||
<p><span style="font-weight: 400;">Shard Manager offers a generic platform for building a sharded application. Users need only to implement a shard state transition interface and express sharding constraints via intent-based specifications. The platform is fully integrated with the rest of the Facebook ecosystem, which hides the complexity of the underlying infrastructure behind a holistic contract and allows our engineers to stay focused on the core business logic of their applications and products.</span></p>
|
||
<p><span style="font-weight: 400;">Shard Manager has evolved since its beginning nine years ago, but the journey is far from complete. We will continue pushing the envelope in providing best-in-class solutions for building sharded services at Facebook.</span></p>
|
||
<p><span style="font-weight: 400;">Despite our success, we are still expanding Shard Manager’s scale and functionalities on multiple fronts. Here are the various challenges we plan to tackle in next few years:</span></p>
|
||
<ol>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">Supporting tens of millions of shards per application to meet the demand from ever-growing, large applications by partitioning the applications internally into smaller independent partitions.</span></li>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">Supporting more complex applications by providing higher modularity and pluggability for users to customize while keeping Shard Manager simple.</span></li>
|
||
<li style="font-weight: 400;"><span style="font-weight: 400;">Simplifying the onboarding experience for the long tail of small, simple applications for which current abstraction is too heavyweight.</span></li>
|
||
</ol>
|
||
<p><span style="font-weight: 400;">By demonstrating the feasibility of building a generic sharding solution, we hope we can foster further discussion and help to collectively advance the field around this cutting-edge problem in the technical community.</span></p>
|
||
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-facebook"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-facebook-16558"
|
||
class="share-facebook sd-button share-icon"
|
||
href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/?share=facebook"
|
||
target="_blank"
|
||
aria-labelledby="sharing-facebook-16558"
|
||
>
|
||
<span id="sharing-facebook-16558" hidden>Share on Facebook (Opens in new window)</span>
|
||
<span>Facebook</span>
|
||
</a></li><li class="share-threads"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-threads-16558"
|
||
class="share-threads sd-button share-icon"
|
||
href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/?share=threads"
|
||
target="_blank"
|
||
aria-labelledby="sharing-threads-16558"
|
||
>
|
||
<span id="sharing-threads-16558" hidden>Share on Threads (Opens in new window)</span>
|
||
<span>Threads</span>
|
||
</a></li><li class="share-jetpack-whatsapp"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-whatsapp-16558"
|
||
class="share-jetpack-whatsapp sd-button share-icon"
|
||
href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/?share=jetpack-whatsapp"
|
||
target="_blank"
|
||
aria-labelledby="sharing-whatsapp-16558"
|
||
>
|
||
<span id="sharing-whatsapp-16558" hidden>Share on WhatsApp (Opens in new window)</span>
|
||
<span>WhatsApp</span>
|
||
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-linkedin-16558"
|
||
class="share-linkedin sd-button share-icon"
|
||
href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/?share=linkedin"
|
||
target="_blank"
|
||
aria-labelledby="sharing-linkedin-16558"
|
||
>
|
||
<span id="sharing-linkedin-16558" hidden>Share on LinkedIn (Opens in new window)</span>
|
||
<span>LinkedIn</span>
|
||
</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-reddit-16558"
|
||
class="share-reddit sd-button share-icon"
|
||
href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/?share=reddit"
|
||
target="_blank"
|
||
aria-labelledby="sharing-reddit-16558"
|
||
>
|
||
<span id="sharing-reddit-16558" hidden>Share on Reddit (Opens in new window)</span>
|
||
<span>Reddit</span>
|
||
</a></li><li class="share-x"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-x-16558"
|
||
class="share-x sd-button share-icon"
|
||
href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/?share=x"
|
||
target="_blank"
|
||
aria-labelledby="sharing-x-16558"
|
||
>
|
||
<span id="sharing-x-16558" hidden>Share on X (Opens in new window)</span>
|
||
<span>X</span>
|
||
</a></li><li class="share-bluesky"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-bluesky-16558"
|
||
class="share-bluesky sd-button share-icon"
|
||
href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/?share=bluesky"
|
||
target="_blank"
|
||
aria-labelledby="sharing-bluesky-16558"
|
||
>
|
||
<span id="sharing-bluesky-16558" hidden>Share on Bluesky (Opens in new window)</span>
|
||
<span>Bluesky</span>
|
||
</a></li><li class="share-mastodon"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-mastodon-16558"
|
||
class="share-mastodon sd-button share-icon"
|
||
href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/?share=mastodon"
|
||
target="_blank"
|
||
aria-labelledby="sharing-mastodon-16558"
|
||
>
|
||
<span id="sharing-mastodon-16558" hidden>Share on Mastodon (Opens in new window)</span>
|
||
<span>Mastodon</span>
|
||
</a></li><li class="share-custom share-custom-hackernews"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-custom-16558"
|
||
class="share-custom share-custom-hackernews sd-button share-icon"
|
||
href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/?share=custom-1699562127"
|
||
target="_blank"
|
||
aria-labelledby="sharing-custom-16558"
|
||
>
|
||
<span id="sharing-custom-16558" hidden>Share on Hacker News (Opens in new window)</span>
|
||
<span style="background-image:url("https://news.ycombinator.com/y18.svg");">Hacker News</span>
|
||
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-email-16558"
|
||
class="share-email sd-button share-icon"
|
||
href="mailto:?subject=%5BShared%20Post%5D%20Scaling%20services%20with%20Shard%20Manager&body=https%3A%2F%2Fengineering.fb.com%2F2020%2F08%2F24%2Fproduction-engineering%2Fscaling-services-with-shard-manager%2F&share=email"
|
||
target="_blank"
|
||
aria-labelledby="sharing-email-16558"
|
||
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you're having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="eb7f986e85" data-email-share-track-url="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/?share=email">
|
||
<span id="sharing-email-16558" hidden>Email a link to a friend (Opens in new window)</span>
|
||
<span>Email</span>
|
||
</a></li><li class="share-end"></li></ul></div></div></div>
|
||
|
||
</div><!-- .entry-content -->
|
||
|
||
|
||
</article><!-- #post-## -->
|
||
|
||
<footer class="entry-footer">
|
||
|
||
|
||
</footer><!-- .entry-footer -->
|
||
|
||
<div class="fb-like"
|
||
data-href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/"
|
||
data-width=""
|
||
data-layout="standard"
|
||
data-action="like"
|
||
data-size="small"
|
||
data-share="true">
|
||
</div>
|
||
|
||
|
||
<div class="related-articles">
|
||
<div class="related-articles__header">
|
||
<h3 class="related-articles__header__headline">Read More in Production Engineering</h3>
|
||
<a
|
||
aria-label="View all articles in this Production Engineering"
|
||
class="related-articles__header__link"
|
||
href="https://engineering.fb.com/category/production-engineering/"
|
||
>
|
||
View All </a>
|
||
</div>
|
||
|
||
<div class="article-grids related-articles__grid">
|
||
<div class="row">
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-16558 type-post status-publish format-standard has-post-thumbnail hentry category-core-infra category-production-engineering fb_content_type-article" id="post-24394">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/Meta-MTIA-300-Hero-final.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Meta MTIA 300 - Hero - final" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-08-24T10:45:52-07:00">
|
||
AUG 24, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/mtia-300-meta-training-chip-built-in-nics/" rel="bookmark" aria-label="Read More about MTIA 300: Meta’s First Training Chip with Built-in NICs and Communication-Offloading Engines">
|
||
MTIA 300: Meta’s First Training Chip with Built-in NICs and Communication-Offloading Engines </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-16558 type-post status-publish format-standard has-post-thumbnail hentry category-core-infra category-production-engineering fb_content_type-article" id="post-23996">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/05/Meta-Mult-stage-Ads-Ranking.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-08-05T12:20:20-07:00">
|
||
AUG 5, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/08/05/ml-applications/from-user-sequences-to-scaling-laws-a-multi-stage-architecture-for-metas-ads-ranking/" rel="bookmark" aria-label="Read More about From User Sequences to Scaling Laws: A Multi-Stage Architecture for Meta’s Ads Ranking">
|
||
From User Sequences to Scaling Laws: A Multi-Stage Architecture for Meta’s Ads Ranking </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-16558 type-post status-publish format-standard has-post-thumbnail hentry category-core-infra category-production-engineering fb_content_type-article" id="post-24150">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Metas-AI-Storage-Blueprint-at-Scale-Hero-1.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-07-01T09:00:36-07:00">
|
||
JUL 1, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/07/01/data-infrastructure/metas-ai-storage-blueprint-at-scale/" rel="bookmark" aria-label="Read More about Meta’s AI Storage Blueprint at Scale">
|
||
Meta’s AI Storage Blueprint at Scale </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-16558 type-post status-publish format-standard has-post-thumbnail hentry category-core-infra category-production-engineering fb_content_type-article" id="post-24128">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/10-Years-Meta-Python-HERO-large.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-06-30T09:00:46-07:00">
|
||
JUN 30, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/06/30/open-source/10-years-of-metas-commitment-to-python/" rel="bookmark" aria-label="Read More about 10 Years of Meta’s Commitment to Python">
|
||
10 Years of Meta’s Commitment to Python </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-16558 type-post status-publish format-standard has-post-thumbnail hentry category-core-infra category-production-engineering fb_content_type-article" id="post-24113">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Meta-Tech-Podcast-How-Meta-Built-Ultra-Narrow-Batteries-for-AI-Glasses-HERO.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-06-23T09:00:38-07:00">
|
||
JUN 23, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/06/23/production-engineering/how-meta-built-ultra-narrow-batteries-for-ai-glasses-meta-tech-podcast/" rel="bookmark" aria-label="Read More about How Meta Engineered Ultra-Narrow Batteries for AI Glasses">
|
||
How Meta Engineered Ultra-Narrow Batteries for AI Glasses </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-16558 type-post status-publish format-standard has-post-thumbnail hentry category-core-infra category-production-engineering fb_content_type-article" id="post-24022">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Validating-PowerLoss-Readiness-Meta-hero.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-06-03T10:00:44-07:00">
|
||
JUN 3, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/06/03/data-center-engineering/lights-out-systems-on-validating-instant-power-loss-readiness/" rel="bookmark" aria-label="Read More about Lights Out, Systems On: Validating Instant Power Loss Readiness">
|
||
Lights Out, Systems On: Validating Instant Power Loss Readiness </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
</div><!-- /.row -->
|
||
</div><!--/.article-grids -->
|
||
|
||
</div><!-- /.related-articles -->
|
||
|
||
|
||
<!-- include the carousel nav here. -->
|
||
|
||
|
||
|
||
|
||
</main><!-- #main -->
|
||
|
||
</div><!-- #primary -->
|
||
|
||
<!-- Do the right sidebar check. -->
|
||
|
||
|
||
|
||
|
||
<div class="col-md-4 widget-area" id="right-sidebar" role="complementary">
|
||
<aside id="fb_related_post_widget-4" class="widget widget_fb_related_post_widget"><h3 class="widget-title">Related Posts</h3><hr>
|
||
<div class="aside-related-projects"><div class="article-grid-redesign">
|
||
<article class="post-13091 post type-post status-publish has-post-thumbnail hentry" id="post-13091">
|
||
<a href="https://engineering.fb.com/2019/06/06/data-center-engineering/twine/" rel="bookmark" aria-label="Read More about Efficient, reliable cluster management at scale with Twine">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2019/06/SystemsHero.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2019/06/SystemsHero.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2019/06/SystemsHero.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Jun 06, 2019">Jun 06, 2019</time></span>
|
||
<h4 class="entry-title">Efficient, reliable cluster management at scale with Twine</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div><div class="article-grid-redesign">
|
||
<article class="post-16510 post type-post status-publish has-post-thumbnail hentry" id="post-16510">
|
||
<a href="https://engineering.fb.com/2020/08/17/production-engineering/async/" rel="bookmark" aria-label="Read More about Asynchronous computing @Facebook: Driving efficiency and developer productivity at Facebook scale">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2020/08/Async-hero-1.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/Async-hero-1.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2020/08/Async-hero-1.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Aug 17, 2020">Aug 17, 2020</time></span>
|
||
<h4 class="entry-title">Asynchronous computing @Facebook: Driving efficiency and developer productivity at Facebook scale</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div><div class="article-grid-redesign">
|
||
<article class="post-12578 post type-post status-publish has-post-thumbnail hentry" id="post-12578">
|
||
<a href="https://engineering.fb.com/2019/03/14/data-center-engineering/accelerating-infrastructure/" rel="bookmark" aria-label="Read More about Accelerating Facebook's infrastructure with application-specific hardware">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2019/03/animation_1920X581_1.gif?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2019/03/animation_1920X581_1.gif?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2019/03/animation_1920X581_1.gif?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Mar 14, 2019">Mar 14, 2019</time></span>
|
||
<h4 class="entry-title">Accelerating Facebook's infrastructure with application-specific hardware</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div></div></aside><aside id="wpb_widget-3" class="widget widget_wpb_widget"><h3 class="widget-title">Related Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2316600655751692/>
|
||
<div class="job-description">
|
||
Machine Learning Engineer
|
||
</div>
|
||
<div class="job-location">
|
||
TEL AVIV, IL
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2887906508264541/>
|
||
<div class="job-description">
|
||
Software Engineer (Leadership) - Product
|
||
</div>
|
||
<div class="job-location">
|
||
LONDON, GB
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
SUNNYVALE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
BELLEVUE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
MENLO PARK, US
|
||
</div>
|
||
</a>
|
||
</li> </ul>
|
||
<a href="https://www.metacareers.com">
|
||
<div class="btn available-positions-btn">
|
||
See All Jobs </div>
|
||
</a>
|
||
</aside>
|
||
</div><!-- #secondary -->
|
||
|
||
|
||
|
||
<!-- <php get_template_part('content-single-side-bar'); ?> -->
|
||
|
||
<!-- Do the right sidebar check. -->
|
||
|
||
</div><!-- Container end -->
|
||
|
||
</div><!-- Wrapper end -->
|
||
|
||
</div>
|
||
|
||
<!-- ******************* The Footer Full-width Widget Area ******************* -->
|
||
|
||
<div class="wrapper wrapper-community-outer">
|
||
<div class="wrapper-community">
|
||
|
||
<div class="community" tabindex="-1">
|
||
|
||
<div class="row">
|
||
<!-- community section is generated through WordPress widget -->
|
||
<div id="wpb_widget-2" class="footer-widget widget_wpb_widget widget-count-3 col-md-4"><h3 class="widget-title">Available Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2316600655751692/>
|
||
<div class="job-description">
|
||
Machine Learning Engineer
|
||
</div>
|
||
<div class="job-location">
|
||
TEL AVIV, IL
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2887906508264541/>
|
||
<div class="job-description">
|
||
Software Engineer (Leadership) - Product
|
||
</div>
|
||
<div class="job-location">
|
||
LONDON, GB
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
SUNNYVALE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
BELLEVUE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
MENLO PARK, US
|
||
</div>
|
||
</a>
|
||
</li> </ul>
|
||
<a href="https://www.metacareers.com">
|
||
<div class="btn available-positions-btn">
|
||
See All Jobs </div>
|
||
</a>
|
||
</div><!-- .footer-widget --><div id="custom_html-2" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Technology at Meta</h3><div class="textwidget custom-html-widget"><ul class="stay-connected-lists">
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-fb-engineering" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Engineering at Meta - X
|
||
</div>
|
||
<a onclick='return twitterFollow("Meta_Engineers")'>
|
||
<button class="btn stay-connected-btn">
|
||
Follow
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-AI" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
AI at Meta
|
||
</div>
|
||
<a href="https://ai.meta.com/blog/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta Quest Blog
|
||
</div>
|
||
<a href="https://www.meta.com/blog/quest/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta for Developers
|
||
</div>
|
||
<a href="https://developers.facebook.com/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-bug-bounty" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta Bug Bounty
|
||
</div>
|
||
<a href="https://bugbounty.meta.com/">
|
||
<button class="btn stay-connected-btn">
|
||
Learn more
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
|
||
<li class="stay-connected-item">
|
||
<img class="stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/rss.png" alt="footer-rss" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
RSS
|
||
</div>
|
||
<a href="https://code.facebook.com/posts/rss/">
|
||
<button class="btn stay-connected-btn">
|
||
Subscribe
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div></div><!-- .footer-widget --><div id="custom_html-3" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Open Source</h3><div class="textwidget custom-html-widget"><p class="opensource-desc">
|
||
Meta believes in building community through open source technology. Explore our latest projects in Artificial Intelligence, Data Infrastructure, Development Tools, Front End, Languages, Platforms, Security, Virtual Reality, and more.
|
||
</p>
|
||
<ul class="opensource-cats">
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/android.png" alt="android" loading="lazy" width="50" height="59">
|
||
<div class="opensource-cap">
|
||
ANDROID
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/ios.png" alt="ios" loading="lazy" width="48" height="58">
|
||
<div class="opensource-cap">
|
||
iOS
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/web.png" alt="web" loading="lazy" width="55" height="49">
|
||
<div class="opensource-cap">
|
||
WEB
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/backend.png" alt="backend" loading="lazy" width="56" height="51">
|
||
<div class="opensource-cap">
|
||
BACKEND
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/hardware.png" alt="hardware" loading="lazy" width="55" height="44">
|
||
<div class="opensource-cap">
|
||
HARDWARE
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
|
||
<button class="btn opensource-btn" onclick="window.location.href='https://opensource.fb.com/'">
|
||
Learn More
|
||
</button>
|
||
|
||
</div></div><!-- .footer-widget -->
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</div><!-- #wrapper-footer-full -->
|
||
|
||
|
||
<div class="wrapper-footer" id="wrapper-footer">
|
||
<div class="inner-container">
|
||
<div class="footer">
|
||
<div class="row">
|
||
<div class="col-md-12">
|
||
<footer class="meta-footer" id="meta-footer">
|
||
<div class="site-info row justify-content-between">
|
||
<div class="col-md-5 meta-logo-container">
|
||
<a href="https://about.facebook.com/">
|
||
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/meta_logo_full.svg" width="89" height="18" alt="Meta" />
|
||
</a>
|
||
<div id="text-2" class="widget widget_text"> <div class="textwidget"><p>Engineering at Meta is a technical news resource for engineers interested in how we solve large-scale technical challenges at Meta.</p>
|
||
</div>
|
||
</div> </div>
|
||
<div class="col-md-6 footer-meta-links-container">
|
||
<ul class="meta-footer-links">
|
||
<li class="footer-link"><a href="https://engineering.fb.com">Home</a></li><li class="footer-link"><a href="https://about.meta.com/">Company Info</a></li><li class="footer-link"><a href="https://www.metacareers.com/?ref=engineering.fb.com">Careers</a></li> </ul>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="footer">
|
||
<div class="row">
|
||
|
||
<div class="col-md-12">
|
||
|
||
<footer class="site-footer" id="colophon">
|
||
<div class="inner-container">
|
||
<div class="site-info row justify-content-between">
|
||
|
||
<div class="col-md-4 copyright-container">
|
||
<div class="copyright"> © 2026 Meta</div>
|
||
</div>
|
||
|
||
<!-- static footer -->
|
||
<div class="col-md-7 footer-links-container">
|
||
<ul class="footer-links">
|
||
<li class="footer-link"><a href="https://www.facebook.com/policies">Terms</a></li><li class="footer-link"><a href="https://www.facebook.com/privacy/policy">Privacy</a></li><li class="footer-link"><a href="/privacy">Cookies</a></li><li class="footer-link"><a href="https://www.facebook.com/help">Help</a></li> </ul>
|
||
</div>
|
||
</div>
|
||
</div><!-- .site-info -->
|
||
|
||
</footer><!-- #colophon -->
|
||
|
||
</div><!--col end -->
|
||
|
||
</div><!-- row end -->
|
||
|
||
</div><!-- //.inner-container -->
|
||
</div><!-- wrapper end -->
|
||
|
||
</div><!-- #page we need this extra closing tag here -->
|
||
|
||
<script type="speculationrules">
|
||
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/code-fb-com/*","/wp-content/themes/understrap/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
|
||
</script>
|
||
<div class="gdprconsent-container" id="GDPRConsentBar">
|
||
<div class="gdprconsent-wrapper">
|
||
<div class="gdprconsent-content">
|
||
<p>To help personalize content, tailor and measure ads and provide a safer experience, we use cookies. By clicking or navigating the site, you agree to allow our collection of information on and off Facebook through cookies. Learn more, including about available controls: <a href="/privacy">Cookie Policy</a></p>
|
||
<div class="gdprconsent-button">
|
||
<button type="button" data-gdpr-accept>Accept</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Twitter universal website tag code -->
|
||
<script>
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
|
||
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
|
||
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
|
||
// Insert Twitter Pixel ID and Standard Event data below
|
||
twq('init','o31u0');
|
||
twq('track','PageView');
|
||
});
|
||
}
|
||
</script>
|
||
<!-- End Twitter universal website tag code -->
|
||
|
||
|
||
<script type="text/javascript">
|
||
window.WPCOM_sharing_counts = {"https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/":16558};
|
||
</script>
|
||
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KZRLGZ8" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
<noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=660742068067366&ev=PageView&noscript=1"/></noscript>
|
||
<script id="child-understrap-scripts-js-extra">
|
||
var loadmore_params = {"restfulURL":"https://engineering.fb.com/wp-json/fb/v1/loadmore","searchText":"","posts":"{\"page\":0,\"year\":2020,\"monthnum\":8,\"day\":24,\"name\":\"scaling-services-with-shard-manager\",\"category_name\":\"production-engineering\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","post_type":"post"};
|
||
var search_params = {"ajaxurl":"https://engineering.fb.com/wp-admin/admin-ajax.php","posts":"{\"page\":0,\"year\":2020,\"monthnum\":8,\"day\":24,\"name\":\"scaling-services-with-shard-manager\",\"category_name\":\"production-engineering\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","article_filter":"2f23ad5ce3"};
|
||
var more_articles = {"next_article":{"label":"Next","permalink":"https://engineering.fb.com/2020/08/31/developer-tools/zookeeper-twine/","title":"Containerizing ZooKeeper with Twine: Powering container orchestration from within","thumbnail":["https://engineering.fb.com/wp-content/uploads/2020/08/ZK-Hero-Wht-100.jpg?w=580&h=326&crop=1",580,326,true]},"prev_article":{"label":"Prev","permalink":"https://engineering.fb.com/2020/08/17/production-engineering/async/","title":"Asynchronous computing @Facebook: Driving efficiency and developer productivity at Facebook scale","thumbnail":["https://engineering.fb.com/wp-content/uploads/2020/08/Async-hero-1.jpg?w=580&h=326&crop=1",580,326,true]}};
|
||
//# sourceURL=child-understrap-scripts-js-extra
|
||
</script>
|
||
<script id="jetpack-facebook-embed-js-extra">
|
||
var jpfbembed = {"appid":"249643311490","locale":"en_US"};
|
||
//# sourceURL=jetpack-facebook-embed-js-extra
|
||
</script>
|
||
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJx9jUEOwiAQAD/kshJjqwfjU0xZtnZpF0iB+H2J8eTB62Qmg68MlGLlWLEurFyQkmeYXceKXkrF0Nkim4ePYFSiCeWA/8se5V2K/qraIG/tKbFg4JonWsEOxuJDIqFrfYPzROxSWoHVsf/+7nqz4+V6HM/DyYY3/BNA4A==" ></script><script id="sharing-js-js-extra">
|
||
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":""};
|
||
//# sourceURL=sharing-js-js-extra
|
||
</script>
|
||
<script id="sharing-js-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/_inc/build/sharedaddy/sharing.min.js?ver=16.1.3"></script>
|
||
<script id="sharing-js-js-after">
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-facebook' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-threads' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-threads' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomthreads', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-linkedin' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-x' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-x' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomx', 'menubar=1,resizable=1,width=600,height=350' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-bluesky' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-bluesky' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcombluesky', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-mastodon' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-mastodon' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcommastodon', 'menubar=1,resizable=1,width=460,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
//# sourceURL=sharing-js-js-after
|
||
</script>
|
||
<script id="video-analytics-js-after">
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
let youtube = document.querySelectorAll('[src*="youtube-nocookie"]');
|
||
for (let i = 0; i < youtube.length; i++) {
|
||
youtube[i].src = youtube[i].src.replace('youtube-nocookie.com', 'youtube.com');
|
||
}
|
||
let vimeo = document.querySelectorAll('[src*="vimeo"]');
|
||
for (let i = 0; i < vimeo.length; i++) {
|
||
vimeo[i].src = vimeo[i].src.replace('&dnt=true', '');
|
||
}
|
||
});
|
||
}
|
||
//# sourceURL=video-analytics-js-after
|
||
</script>
|
||
<script id="wp-emoji-settings" type="application/json">
|
||
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://engineering.fb.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
|
||
</script>
|
||
<script type="module">
|
||
/*! This file is auto-generated */
|
||
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
|
||
//# sourceURL=https://engineering.fb.com/wp-includes/js/wp-emoji-loader.min.js
|
||
</script>
|
||
|
||
</body>
|
||
|
||
</html>
|