Files
nexus/sreweekly/articles/321/09-belljar-a-new-framework-for-testing-system-recoverability-at-scale.html
2026-09-12 17:23:01 +08:00

1221 lines
114 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-title" content="Engineering at Meta - Engineering at Meta Blog">
<meta property="fb:app_id" content="1425766027653270" />
<link rel="profile" href="http://gmpg.org/xfn/11">
<link rel="pingback" href="https://engineering.fb.com/xmlrpc.php">
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
<script async src="https://engineering.fb.com/wp-content/themes/code-fb-com/dist/js/optimize.min.js"></script>
<!-- This site is optimized with the Yoast SEO Premium plugin v19.3 (Yoast SEO v27.3) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
<title>BellJar: A new framework for testing system recoverability at scale - Engineering at Meta</title>
<meta name="description" content="BellJar is a framework that allows for exercising infrastructure code under worst-case conditions to uncover the relationships that matter during recovery." />
<link rel="canonical" href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/" />
<meta name="twitter:label1" content="Written by" />
<meta name="twitter:data1" content="Christopher Bunn, Jie Huang" />
<meta name="twitter:label2" content="Est. reading time" />
<meta name="twitter:data2" content="17 minutes" />
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/#article","isPartOf":{"@id":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/"},"author":[{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/ac9bb451cb7fb69e994ee4da19de09d5"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/bb1d2b02b8561a960aa7bcd3128d89b4"}],"headline":"BellJar: A new framework for testing system recoverability at scale","datePublished":"2022-05-05T19:00:52+00:00","dateModified":"2022-05-05T23:45:29+00:00","mainEntityOfPage":{"@id":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/"},"wordCount":3607,"publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"image":{"@id":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/05\/BellJar_Titleslide.png","articleSection":["Data Infrastructure","DevInfra"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/","url":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/","name":"BellJar: A new framework for testing system recoverability at scale - Engineering at Meta","isPartOf":{"@id":"https:\/\/engineering.fb.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/#primaryimage"},"image":{"@id":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/05\/BellJar_Titleslide.png","datePublished":"2022-05-05T19:00:52+00:00","dateModified":"2022-05-05T23:45:29+00:00","description":"BellJar is a framework that allows for exercising infrastructure code under worst-case conditions to uncover the relationships that matter during recovery.","breadcrumb":{"@id":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/#primaryimage","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/05\/BellJar_Titleslide.png","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/05\/BellJar_Titleslide.png","width":1594,"height":897},{"@type":"BreadcrumbList","@id":"https:\/\/engineering.fb.com\/2022\/05\/05\/developer-tools\/belljar\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/engineering.fb.com\/"},{"@type":"ListItem","position":2,"name":"BellJar: A new framework for testing system recoverability at scale"}]},{"@type":"WebSite","@id":"https:\/\/engineering.fb.com\/#website","url":"https:\/\/engineering.fb.com\/","name":"Engineering at Meta","description":"Engineering at Meta Blog","publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/engineering.fb.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/engineering.fb.com\/#organization","name":"Meta","url":"https:\/\/engineering.fb.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","width":29011,"height":12501,"caption":"Meta"},"image":{"@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/Engineering\/","https:\/\/x.com\/fb_engineering"]},[],{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/ac9bb451cb7fb69e994ee4da19de09d5","name":"Christopher Bunn","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Christopher Bunn"},"url":"https:\/\/engineering.fb.com\/author\/christopher-bunn\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/bb1d2b02b8561a960aa7bcd3128d89b4","name":"Jie Huang","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Jie Huang"},"url":"https:\/\/engineering.fb.com\/author\/jie-huang\/"}]}</script>
<!-- / Yoast SEO Premium plugin. -->
<link rel='dns-prefetch' href='//secure.gravatar.com' />
<link rel='dns-prefetch' href='//v0.wordpress.com' />
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta &raquo; Feed" href="https://engineering.fb.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta &raquo; Comments Feed" href="https://engineering.fb.com/comments/feed/" />
<script async src="https://www.googleoptimize.com/optimize.js?id=OPT-TPNDQC5" onerror="dataLayer.hide.end && dataLayer.hide.end()"></script>
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2022%2F05%2F05%2Fdeveloper-tools%2Fbelljar%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2022%2F05%2F05%2Fdeveloper-tools%2Fbelljar%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='all-css-12' href='https://engineering.fb.com/_static/??-eJyNjUsOwjAMRC9EaiqghQXiKFVqhzSQn+pEiNuTFJBAbNjZTzNv4BYFBp+UTxBt1sYzFKQpzpVz5WQ4QSUv0CDzCj6KaVJOMWAgJc5jwe7ZKTnAyVgSS6Jxxv90XRbv3YtKUeJVtF3TgguUbZHyJGdFkui+nMbrvxWD8QhjLvvAAY20wgYd+OupspM7tv3+sO533Wb7AImQY4U=' type='text/css' media='all' />
<script id="gdprconsent-js" src="https://engineering.fb.com/wp-content/plugins/wp-gdpr-consent/dist/gdprconsent.js?ver=2.0.0"></script>
<script id="gdprconsent-js-after">
if (window.gdprSafeTrack) {
window.gdprSafeTrack('https://www.googletagmanager.com/gtag/js?id=391063774');
window.gdprSafeTrack(function() {
var google_analytics_g4 = decodeURIComponent( '391063774' );
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', google_analytics_g4);
});
}
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
var google_tag_manager = decodeURIComponent( 'GTM-KZRLGZ8' );
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer',google_tag_manager);
});
}
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
var facebook_pixel = decodeURIComponent( '660742068067366' );
!function(f,b,e,v,n,t,s)
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
n.queue=[];t=b.createElement(e);t.async=!0;
t.src=v;s=b.getElementsByTagName(e)[0];
s.parentNode.insertBefore(t,s)}(window, document,'script',
'https://connect.facebook.net/en_US/fbevents.js');
fbq('init', facebook_pixel);
fbq('track', 'PageView');
});
}
//# sourceURL=gdprconsent-js-after
</script>
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJzTLy/QzcxLzilNSS3WzwKiwtLUokoopZebmaeXVayjj0+Rbm5melFiSSpUsX2uraG5hbmZgYmloUkWAK+3Iic=" ></script><script type="text/javascript" src="https://engineering.fb.com/wp-content/themes/understrap/js/popper.min.js?m=1773259670g" ></script><link rel="https://api.w.org/" href="https://engineering.fb.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.fb.com/wp-json/wp/v2/posts/18735" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.fb.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress 7.1" />
<link rel='shortlink' href='https://wp.me/sa0Lhq-belljar' />
<link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/Optimistic_Display_W_Md.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="BellJar: A new framework for testing system recoverability at scale" />
<meta property="og:url" content="https://engineering.fb.com/2022/05/05/developer-tools/belljar/" />
<meta property="og:description" content="Building infrastructure that can easily recover from outages, particularly outages involving adjacent infrastructure, too often becomes a murky exploration of nuanced fate-sharing between systems. …" />
<meta property="article:published_time" content="2022-05-05T19:00:52+00:00" />
<meta property="article:modified_time" content="2022-05-05T23:45:29+00:00" />
<meta property="og:site_name" content="Engineering at Meta" />
<meta property="og:image" content="https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Titleslide.png" />
<meta property="og:image:width" content="1594" />
<meta property="og:image:height" content="897" />
<meta property="og:image:alt" content="" />
<meta property="og:locale" content="en_US" />
<meta name="twitter:text:title" content="BellJar: A new framework for testing system recoverability at scale" />
<meta name="twitter:image" content="https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Titleslide.png?w=640" />
<meta name="twitter:card" content="summary_large_image" />
<!-- End Jetpack Open Graph Tags -->
<style id="wp-custom-css">
.table-of-contents {
position: sticky;
top: 20px;
background: #fff;
padding: 1em;
border: 1px solid #ddd;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
max-width: 250px;
}
</style>
<link rel="shortcut icon" href="https://engineering.fb.com/wp-content/themes/code-fb-com/favicon.ico" />
</head>
<body class="wp-singular post-template-default single single-post postid-18735 single-format-standard wp-theme-understrap wp-child-theme-code-fb-com group-blog">
<div class="hfeed site" id="page">
<!-- ******************* The Navbar Area ******************* -->
<div class="wrapper-fluid wrapper-navbar" id="wrapper-navbar">
<a class="skip-link screen-reader-text sr-only" href="#content">
Skip to content </a>
<nav class="navbar navbar-expand-std">
<div class="nav-container">
<!-- Your site title as branding in the menu -->
<div class="nav-top-row">
<h1 class="navbar-brand mb-0">
<a rel="home" href="https://engineering.fb.com/" title="Engineering at Meta">
<img class="navbar-brand-logo" src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/logo-meta.svg" class="navbar-brand-logo" alt="Engineering at Meta" width="215" height="40" loading="lazy">
</a>
</h1>
<!-- end custom logo -->
<label for="search" class="screen-reader-text">Search this site</label>
<input type="text" id="search" name="search" class="form-control-lg navbar-search" autocomplete="off">
<button class="navbar-toggler navbar-searchbar-toggler"
type="button"
data-toggle="collapse"
data-target="#searchbarDropdown"
aria-controls="searchbarDropdown"
aria-expanded="false"
aria-label="Toggle navigation">
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/icon-search.svg" class="navbar-mobile-search-icon" loading="lazy" width="24" height="25" />
</button>
<!-- desktop language menu (mobile language menu is implemented via jQuery)-->
<div class="notranslate nice-select navbar-language form-control">
</div>
<button class="navbar-toggler collapsed navbar-menu-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<div class="navbar-border"></div>
<!-- The WordPress Menu goes here -->
<div id="navbarNavDropdown" class="collapse navbar-collapse"><ul id="main-menu" class="navbar-nav"><li id="menu-item-60" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-60 dropdown"><a title="Open Source" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Open Source <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-12330" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-12330"><a title="Open Source" href="https://engineering.fb.com/category/open-source/" class="nav-link">Open Source</a></li>
<li id="menu-item-12331" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-12331"><a title="Meta Open Source" href="https://opensource.fb.com" class="nav-link">Meta Open Source</a></li>
</ul>
</li>
<li id="menu-item-2686" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2686 dropdown"><a title="Platforms" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Platforms <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-65" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-65"><a title="Android" href="https://engineering.fb.com/category/android/" class="nav-link">Android</a></li>
<li id="menu-item-66" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-66"><a title="iOS" href="https://engineering.fb.com/category/ios/" class="nav-link">iOS</a></li>
<li id="menu-item-67" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-67"><a title="Web" href="https://engineering.fb.com/category/web/" class="nav-link">Web</a></li>
</ul>
</li>
<li id="menu-item-2682" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2682 dropdown"><a title="Infrastructure Systems" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Infrastructure Systems <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-218" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-218"><a title="Core Infra" href="https://engineering.fb.com/category/core-infra/" class="nav-link">Core Infra</a></li>
<li id="menu-item-219" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-219"><a title="Data Infrastructure" href="https://engineering.fb.com/category/data-infrastructure/" class="nav-link">Data Infrastructure</a></li>
<li id="menu-item-220" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-220"><a title="DevInfra" href="https://engineering.fb.com/category/developer-tools/" class="nav-link">DevInfra</a></li>
<li id="menu-item-226" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-226"><a title="Production Engineering" href="https://engineering.fb.com/category/production-engineering/" class="nav-link">Production Engineering</a></li>
<li id="menu-item-227" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-227"><a title="Security &amp; Privacy" href="https://engineering.fb.com/category/security/" class="nav-link">Security &amp; Privacy</a></li>
<li id="menu-item-18607" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18607"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/systems-infrastructure/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2683" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2683 dropdown"><a title="Physical Infrastructure" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Physical Infrastructure <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-215" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-215"><a title="Connectivity" href="https://engineering.fb.com/category/connectivity/" class="nav-link">Connectivity</a></li>
<li id="menu-item-216" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-216"><a title="Data Center Engineering" href="https://engineering.fb.com/category/data-center-engineering/" class="nav-link">Data Center Engineering</a></li>
<li id="menu-item-217" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-217"><a title="Networking &amp; Traffic" href="https://engineering.fb.com/category/networking-traffic/" class="nav-link">Networking &amp; Traffic</a></li>
<li id="menu-item-18608" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18608"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/networking-connectivity/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2684" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2684 dropdown"><a title="Video Engineering &amp; AR/VR" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Video Engineering &amp; AR/VR <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-225" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-225"><a title="Video Engineering" href="https://engineering.fb.com/category/video-engineering/" class="nav-link">Video Engineering</a></li>
<li id="menu-item-223" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-223"><a title="Virtual Reality" href="https://engineering.fb.com/category/virtual-reality/" class="nav-link">Virtual Reality</a></li>
<li id="menu-item-13388" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13388"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/augmented-reality-virtual-reality/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2685" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2685 dropdown"><a title="Artificial Intelligence" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Artificial Intelligence <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-222" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-222"><a title="ML Applications" href="https://engineering.fb.com/category/ml-applications/" class="nav-link">ML Applications</a></li>
<li id="menu-item-221" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-221"><a title="AI Research" href="https://engineering.fb.com/category/ai-research/" class="nav-link">AI Research</a></li>
<li id="menu-item-13387" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13387"><a title="Research Publications" href="https://ai.facebook.com/results/?content_types%5B0%5D=publication" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-228" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-228"><a title="Watch Videos" href="/videos" class="nav-link">Watch Videos</a></li>
</ul></div> </div><!-- .container -->
<div class="search-container collapse" id="searchbarDropdown">
<div class="searchbar-search">
<div class="searchbar-container">
<div class="navbar-mobile-search-margin"></div>
<input type="text" name="search" placeholder="Search" class="form-control-lg navbar-mobile-search" autofocus autocomplete="off">
<button class="searchbar-clear">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
</div>
</div>
</nav><!-- .site-navigation -->
</div><!-- .wrapper-navbar end -->
<div class="wrapper" id="single-wrapper">
<div class="single-container row" id="content" tabindex="-1">
<!-- Do the left sidebar check. -->
<div class="content-area" id="primary">
<main class="site-main" id="main">
<article class="post-18735 post type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure category-developer-tools fb_content_type-article" id="post-18735">
<header class="entry-header">
<div class="entry-meta">
POSTED ON <span class="posted-on entry-date"><time class="published updated" datetime="2022-05-05">MAY 5, 2022</time></span> TO <span class="cat-links"><a href="https://engineering.fb.com/category/data-infrastructure/" rel="category tag">Data Infrastructure</a>, <a href="https://engineering.fb.com/category/developer-tools/" rel="category tag">DevInfra</a></span>
<h1 class="entry-title"><div>BellJar: A new framework for testing system recoverability at scale</div></h1> </div><!-- .entry-meta -->
</header><!-- .entry-header -->
<figure id="post-feat-image-container">
<img width="1594" height="897" src="https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Titleslide.png" class="wp-post-image" alt="" title="BellJar_Titleslide" /> </figure>
<div class="entry-authors-social">
<div class="entry-authors">
By <a href="https://engineering.fb.com/author/christopher-bunn/" title="Posts by Christopher Bunn" class="author url fn" rel="author">Christopher Bunn</a>, <a href="https://engineering.fb.com/author/jie-huang/" title="Posts by Jie Huang" class="author url fn" rel="author">Jie Huang</a> </div>
</div>
<div class="entry-content">
<p><span style="font-weight: 400;">Building infrastructure that can easily recover from outages, particularly outages involving adjacent infrastructure, too often becomes a murky exploration of nuanced fate-sharing between systems. Untangling dependencies and uncovering side effects of unavailability has historically been time-consuming work.</span></p>
<p><span style="font-weight: 400;">A lack of great tooling built for this, and the rarity of infrastructure outages, makes reasoning about them difficult. How far will the outage extend? And which available mitigations can get things back online? Bootstrapping blockers and circular dependencies present real concerns we cannot put to bed with theory or design documents alone. The lower in the stack we go, the broader the impact of such outages could be. Purpose-built tooling is important for safely experimenting with relationships between systems low in the stack.</span></p>
<p><span style="font-weight: 400;">BellJar is a new framework we’ve developed at Meta to strip away the mystery and subtlety that has plagued efforts to build recoverable infrastructure. It gives us a simple way to examine how layers in our stack recover from the worst outages imaginable. BellJar has become a powerful, flexible tool we can use to prove that our infrastructure code works the way we expect it to. None of us wants to exercise the recovery strategies, worst-case constrained behaviors, or emergency tooling that BellJar helps us curate. At the same time, careful preparation like this has helped us make our systems more resilient.</span></p>
<div class="fb-video" data-allowfullscreen="true" data-href="https://www.facebook.com/watch/?v=667732707833046" style="background-color: #fff; display: inline-block;"></div>
<h2><span style="font-weight: 400;">Pragmatic coupling between systems: curation over isolation</span></h2>
<p><span style="font-weight: 400;">With an infrastructure at Meta’s scale, where software may span over dozens of data centers and millions of machines, tasks that would be trivial on a small number of servers (such as changing a configuration value) simply cannot be performed manually. The layers of infrastructure we build abstract away many of the difficult parts of running software at scale so that application developers can focus on their system’s unique offerings. And reusing this common infrastructure solves these problems once, universally, for all our service owners. </span></p>
<p><span style="font-weight: 400;">However, the reach of these foundational systems also means that outages could potentially impact a wide swath of other software. Herein lies the trade-off between dependency and easy disaster recovery that infra engineers have to consider. The thinking goes that if you build code that doesn’t need any external systems, then recovery from all sorts of failures should be quick.</span></p>
<p><span style="font-weight: 400;">But with time and growth, the practicalities of large-scale production environments bear down on this simplistic model. All this infrastructure amounts to regular software. </span><span style="font-weight: 400;">Lik</span><span style="font-weight: 400;">e the services it powers, it needs to run somewhere, receive upgrades, scale up or down, expose configuration knobs, store durable state, and communicate with other services. Rebuilding each of these capabilities to ensure perfect isolation for each infrastructure system gets impractical. In many cases, leaning on shared infrastructure to solve these problems rather than reinventing the wheel makes sense. This is especially true at scale, where edge cases can be difficult to get right. We don’t want every infrastructure team to build their own entire support stack from scratch.</span></p>
<p><span style="font-weight: 400;">Yet without a solid understanding of how our systems connect, we can find ourselves in one of two traps: building duplicative support systems in pursuit of overkill decoupling, or allowing our systems to become intertwined in ways that jeopardize their resiliency. Without good visibility and rigor, systems can experience a mix of both as engineers address the low-criticality dependencies they know about. This can create undetected, and therefore unaddressed, circular dependencies.</span></p>
<figure id="attachment_18739" aria-describedby="caption-attachment-18739" style="width: 1024px" class="wp-caption alignnone"><img decoding="async" class="size-large wp-image-18739" src="https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image4.png?w=1024" alt="BellJar" width="1024" height="576" srcset="https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image4.png 1999w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image4.png?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image4.png?resize=916,516 916w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image4.png?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image4.png?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image4.png?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image4.png?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image4.png?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-18739" class="wp-caption-text">Even the lowest-dependency systems, like our Apache ZooKeeper deployments, lean on a wide array of supporting code.</figcaption></figure>
<p><span style="font-weight: 400;">Without structured, rigorous controls in place, a mishmash of uncurated relationships between systems tends to grow unchecked. This then creates coupling between systems and complexity that can prove difficult to reverse. </span></p>
<h2><span style="font-weight: 400;">Recoverability as the design constraint</span></h2>
<p><span style="font-weight: 400;">So how do we avoid senseless duplication while limiting the coupling between systems that can present real operational problems? By focusing on a clear constraint — recoverability, the requirement that whenever outages strike our infrastructure systems, we have strong confidence that our infrastructure can return to a healthy state in a short amount of time.</span></p>
<p><span style="font-weight: 400;">But to analyze cross-system relationships through the lens of recoverability, we need tooling that exposes, tests, and catalogs the relationships buried in our millions of lines of code. BellJar is a framework for exercising infrastructure code under worst-case conditions to uncover the relationships that matter during recovery.</span></p>
<p><span style="font-weight: 400;">Outages can manifest in many forms. For our purposes, we like to focus on a broad, prototypical outage that presents a categorical failure of common infrastructure systems. Using recoverability as the objective allows us to bring tractability to the problem of curated coupling in several ways:</span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><b>Practicality:</b><span style="font-weight: 400;"> Recoverability provides an alternative to the impractical, absolutist position that requires complete isolation between all infra systems. If some coupling between two systems does not interfere with recovery capabilities of our infrastructure, such coupling may be acceptable.</span></li>
<li style="font-weight: 400;" aria-level="1"><b>Characterization:</b><span style="font-weight: 400;"> Recoverability lets us focus on a specific type of intersystem relationship — the relationships required to get a system up to minimally good health. Doing so shrinks the universe of cross-system connections that we need to reason about.</span></li>
<li style="font-weight: 400;" aria-level="1"><b>Scope:</b><span style="font-weight: 400;"> Rather than attempt to optimize the dependency graph for our entire infrastructure at once, we can zoom in on individual systems. By considering only their adjacent supporting software, these become the building blocks of recovery.</span></li>
<li style="font-weight: 400;" aria-level="1"><b>Additive:</b><span style="font-weight: 400;"> A common approach to decoupling systems asks, “What happens to system </span><span style="font-weight: 400;">Foo</span><span style="font-weight: 400;"> if we take away access to system </span><span style="font-weight: 400;">Bar</span><span style="font-weight: 400;">?” However, when your production environment contains thousands of possible dependencies, even enumerating all such test permutations can become impossible. By contrast, a recovery analysis asks the inverse question: “What is everything </span><span style="font-weight: 400;">Foo</span><span style="font-weight: 400;"> needs to reach a minimally healthy state?”</span></li>
<li style="font-weight: 400;" aria-level="1"><b>Testability:</b><span style="font-weight: 400;"> Recovery scenarios offer real, verifiable conditions we can assert programmatically. While outages manifest in many shapes and sizes, in each case, recovery is a Boolean result — either recovery is successful or it isn’t.</span></li>
</ul>
<h2><span style="font-weight: 400;">Examining system recoveries with a total outage in a vacuum</span></h2>
<p><span style="font-weight: 400;">BellJar presents infra teams with an experimentation environment that is indistinguishable from production, with one big difference. It exists vacuum-sealed away from all production systems by default. Such an environment renders the remote systems and local processes that satisfy even basic functionality (configuration, service discovery, request routing, database access, key/value storage systems) unavailable. In this way, the environment presents hardware in a state we’d find during a total outage of Meta infrastructure. Everything but basic operating system functionality is broken.</span></p>
<p><span style="font-weight: 400;">To produce this environment, we lean heavily on our virtualization infrastructure and fault injection systems. Each instance of a BellJar environment is ephemeral, composed of a set of virtual machines (VMs) that constantly rotate through a lease-wipe-update cycle. Before we hand the machines to a BellJar user, we inject an enormously disruptive blanket of faults that blackhole traffic over all network interfaces, including </span><span style="font-weight: 400;">loopback</span><span style="font-weight: 400;">. Our fault injection engine affords us allowlist-style disruptions we can stack against VMs. They disrupt our systems at the IP layer, at the application’s network layer, or on a per-process or container basis.</span></p>
<figure id="attachment_18738" aria-describedby="caption-attachment-18738" style="width: 1024px" class="wp-caption alignnone"><img decoding="async" class="size-large wp-image-18738" src="https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image3.png?w=1024" alt="BellJar" width="1024" height="576" srcset="https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image3.png 1999w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image3.png?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image3.png?resize=916,516 916w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image3.png?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image3.png?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image3.png?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image3.png?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_image3.png?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-18738" class="wp-caption-text">BellJar allows us to build tightly tailored test environments composed only of select production systems.</figcaption></figure>
<p><span style="font-weight: 400;">BellJar’s API gives operators the ability to customize which production capabilities they want to leak back into this vacuum through an additive allowlisting scheme. Need DNS? We’ll allow it. Need access to a specific storage cluster? Just tell BellJar the name of the cluster. Want access to the read-only pathway for a certain binary package backend? You got it.</span></p>
<p><span style="font-weight: 400;">Through granular, checkbox-style allowlisting, the operator can customize the limited set of remote endpoints, local daemons, and libraries they want to earmark as “healthy.” The resulting environment presents a severe outage scenario per the operator’s choosing. But within the environment, the operator can describe and test their own system’s minimally viable recovery requirements and startup procedure, from empty hardware to healthy recovery.</span></p>
<h2><span style="font-weight: 400;">Making each recovery from a unique recipe</span></h2>
<p><span style="font-weight: 400;">This approach reveals several unique ingredients in each BellJar test. Often, teams will define a set of different recovery scenarios for a single piece of infrastructure. Each describes its own combination of the following:</span></p>
<ol>
<li style="font-weight: 400;" aria-level="1"><b>Service under test:</b><span style="font-weight: 400;"> This might be a container scheduler service, a fleetwide config distribution system, a low-dependency coordination back end, a certificate authority, or any number of other infrastructure components we wish to inspect. Most, but not all, of these services run in containers.</span></li>
<li style="font-weight: 400;" aria-level="1"><b>Hardware: </b><span style="font-weight: 400;">Different systems run on different hardware setups. We compose the test environment accordingly.</span></li>
<li style="font-weight: 400;" aria-level="1"><b>Recovery strategy:</b><span style="font-weight: 400;"> Each system has a recovery runbook that its operators expect to follow to bring up the system from scratch on fresh hardware.</span></li>
<li style="font-weight: 400;" aria-level="1"><b>Validation criteria:</b><span style="font-weight: 400;"> The criteria that indicates whether a recovery strategy has produced a healthy system.</span></li>
<li style="font-weight: 400;" aria-level="1"><b>Tooling:</b><span style="font-weight: 400;"> The steps of those recovery runbooks likely involve system-specific tooling that operators will reach for during outage mitigation.</span></li>
<li style="font-weight: 400;" aria-level="1"><b>Recovery conditions:</b><span style="font-weight: 400;"> Most systems require something within our infrastructure to be healthy before they can recover. An allowlist captures this set of assumptions, enumerating each production capability the recovery environment must provide before we can expect it to succeed. </span></li>
</ol>
<p><span style="font-weight: 400;">All these ingredients can be assembled to define a single BellJar test, which the service owner writes with a few lines of Python.</span></p>
<h2><span style="font-weight: 400;">Code conquers lore </span></h2>
<p><span style="font-weight: 400;">Since we can define each of the above inputs programmatically, we can code a fully repeatable test that distills all the assertions a service’s operators want to prove about its ability to recover from failure.</span></p>
<p><span style="font-weight: 400;">Prior to BellJar, we’ve had to scour wiki documentation, interview a team’s seasoned engineers, and spelunk through commit histories — all to understand whether </span><span style="font-weight: 400;">System A</span><span style="font-weight: 400;"> or </span><span style="font-weight: 400;">System B</span><span style="font-weight: 400;"> needed to be recovered first if both were brought offline. Exposing recovery-blocking cyclic dependencies often relied on mining through a team’s tribal knowledge, outdated design documents, or code in multiple programming languages.</span></p>
<p><span style="font-weight: 400;">Now BellJar tests the operators’ assumptions, with a Boolean result. </span><span style="font-weight: 400;">True</span><span style="font-weight: 400;"> means the service can be recovered with the runbook and tooling you defined, under any outage in which at least the test’s recovery conditions are met. And we can deploy a service’s BellJar test in its</span><a href="https://engineering.fb.com/2021/10/20/developer-tools/autonomous-testing/"> <span style="font-weight: 400;">CI/CD pipeline</span></a><span style="font-weight: 400;"> to assert recoverability with every new release candidate before it reaches production.</span></p>
<figure id="attachment_18767" aria-describedby="caption-attachment-18767" style="width: 1024px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="size-large wp-image-18767" src="https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Code.png?w=1024" alt="" width="1024" height="573" srcset="https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Code.png 2048w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Code.png?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Code.png?resize=916,513 916w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Code.png?resize=768,430 768w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Code.png?resize=1024,573 1024w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Code.png?resize=1536,860 1536w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Code.png?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2022/05/BellJar_Code.png?resize=192,107 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-18767" class="wp-caption-text">Users describe the capabilities afforded to their test environment alongside the recovery runbook steps they wish to exercise.</figcaption></figure>
<p><span style="font-weight: 400;">For the individual service owner, the ability to generate a codified, repeatable recovery test opens doors to two exciting prospects:</span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><b>Move faster more safely</b><span style="font-weight: 400;">: If you’ve ever been reluctant to change an arcane setting, retry policy, or failover mechanism, it may have been because you’re unsure how it impacts your disaster recovery posture. Unsure whether you can safely use a library’s new feature without jeopardizing your service’s “low dependency” status? BellJar makes changes like these a no-brainer. Just make the change and post the diff. The automated BellJar test will confirm whether your recovery assertions still hold.</span></li>
<li style="font-weight: 400;" aria-level="1"><b>Actively reduce coupling:</b><span style="font-weight: 400;"> Many service owners discover surprising dependencies they didn’t know existed when they onboard to BellJar, so we’ve learned to onboard new systems gradually. First, the service owner defines their recovery strategy in an environment with all production capabilities available. Then we gradually pare down the recovery conditions allowlist to the bare minimum set of dependencies required for successful results. Through an iterative approach like this, teams uncover cross-system dependency liabilities. They then work to iteratively eliminate those that pose circular bootstrapping risks.</span></li>
</ul>
<h2><span style="font-weight: 400;">Multisystem knock-on effects</span></h2>
<p><span style="font-weight: 400;">Wide coverage reveals cross-system details that were previously hard to understand. As a result, the benefits of the BellJar system multiply when we have many infrastructure service owners using it to explore their recovery requirements.</span></p>
<h3><span style="font-weight: 400;">The recovery graph</span></h3>
<p><span style="font-weight: 400;">BellJar allows teams to focus on the first-hop requirements their services demand for a successful recovery. In doing so, we intentionally do not expect each service owner to understand the full depth of Meta’s entire dependency graph. However, limiting service owners to first-hop dependencies means BellJar alone does not provide them with the full picture needed to find transitive dependency cycles in our stack.</span></p>
<p><span style="font-weight: 400;">Now that we’re onboarding a broad collection of services, we can begin to assemble a large, multi-system recovery graph from the individual puzzle pieces provided by each service owner’s efforts. We provide the same prompt to each service: “What does it take to bring your service online on an empty set of hardware?” By doing so, BellJar helps us generate the composable building blocks we need to tie together recovery requirements of services separated by organizational or time zone barriers. Security, networking, database, storage, and other foundational services express their portions of our recovery graph as code that lends itself to programmatic analysis. We’re finding that this recovery-specific graph does not match the runtime relationships we uncover through distributed tracing alone. </span></p>
<h3><span style="font-weight: 400;">Tests as documentation</span></h3>
<p><span style="font-weight: 400;">Service owners describe their recovery strategy (the runbook used to perform the service’s recovery) as the set of steps a human would need to invoke during disaster recovery. This means a test includes the shell commands an operator will invoke to bootstrap things across otherwise empty machines. Typically, these commands reach for powerful toolkits or automation controls.</span></p>
<p><span style="font-weight: 400;">Codifying a human-oriented recovery runbook inside the BellJar test framework may seem strange at first. But human intervention is typically the first step in addressing the disasters we’re planning for. By capturing this runbook for each system, whether it’s a single command or an exceptionally complex series of steps, we can </span><i><span style="font-weight: 400;">automatically</span></i><span style="font-weight: 400;"> generate the human-friendly documentation operators typically have had to manually maintain on wikis or offline reference manuals.</span></p>
<p><span style="font-weight: 400;">Every time a service’s BellJar test passes, the corresponding recovery documentation gets regenerated in human-friendly HTML. This eliminates the staleness and toil we tend to associate with traditional runbooks.</span></p>
<h3><span style="font-weight: 400;">Federated recovery and healthy ownership</span></h3>
<p><span style="font-weight: 400;">Consider a foundation composed of scores of infrastructure systems. It is not scalable to assume we can task a single dedicated team, even a large one, with broadly solving all our worst-case recovery needs. At Meta, we do have a number of teams working on cross-cutting reliability efforts, protecting us against RPC overload, motivating</span><a href="https://engineering.fb.com/2020/09/08/data-center-engineering/fault-tolerance-through-optimal-workload-placement/"> <span style="font-weight: 400;">safe deployment</span></a><span style="font-weight: 400;"> in light of unexpected capacity loss, and equipping teams with powerful fault injection tools. We also have domain experts focused on multisystem recoveries of portions of our stack, such as intra-data center networking or container management.</span></p>
<p><span style="font-weight: 400;">However, we’d best avoid a model in which service owners feel they can offload the complexities of disaster recovery to an external team as “someone else’s problem.” The projects mentioned above achieve success not because they absolve service owners of this responsibility, but because they help service owners better solve specific aspects of disaster readiness.</span></p>
<p><span style="font-weight: 400;">BellJar takes this model a step further by federating service recovery validation out to service owners’ development pipelines. Rather than outsource to some separate team, the service owner designs their recovery strategy. They then test this strategy. And they receive the failing test signal when their strategy gets invalidated.</span></p>
<p><span style="font-weight: 400;">By moving recovery into the service owner’s release cycle, BellJar motivates developers to consider how their system can recover simply and reliably.</span></p>
<h3><span style="font-weight: 400;">Common tooling</span></h3>
<p><span style="font-weight: 400;">Until recently, service owners who have been especially interested in improving their disaster readiness have found themselves building a lot of bespoke recovery tooling. Without a standard approach that’s shared by many teams, the typical service owner discovers and patches gaps in recovery tooling on an ad-hoc basis. Need a way to spin up a container in a pinch? A way to distribute failsafe routing information? Tools to retrieve binaries from safe storage? We’ve found teams reinventing similar tools in silos because of the relative difficulty in discovering common recovery needs across team boundaries.</span></p>
<p><span style="font-weight: 400;">By exposing recovery runbooks in a universal format that can be collected in a single place, BellJar is helping us easily identify the common barriers that teams need standard solutions for. The team building BellJar has consulted with dozens of service owners. As a result, we’ve helped produce standard solutions like:</span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">A common toolkit for constructing low-level containers in the absence of a healthy container orchestrator.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Fleetwide support for emergency binary distribution that bypasses most of our common packaging infrastructure.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Secure access control over emergency tooling that cooperates with service owners’ existing ACLs.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Centralized collection and monitoring of the inputs (metadata and container specifications) needed for recovery tooling.</span></li>
</ul>
<p><span style="font-weight: 400;">By building well-tested, shared solutions for these common problems, we can ensure that those tools get first class treatment under a dedicated team’s ownership.</span></p>
<h2><span style="font-weight: 400;">Common patterns across systems</span></h2>
<p><span style="font-weight: 400;">Thanks to BellJar’s uniquely broad view of our infrastructure, we’ve also been able to distill some pretty interesting patterns that teams have had to grapple with when designing for recovery.</span></p>
<h3><span style="font-weight: 400;">The Public Key Infrastructure foundation</span></h3>
<p><span style="font-weight: 400;">Access controls and TLS define every action that both human operators and automated systems can and cannot take. Security becomes important during disaster recovery, just as in a steady state. This means that systems like certificate authorities are important to any recovery strategy. In an environment where every container requires an x.509 certificate from a trusted issuer, removing dependency cycles for containerized security services becomes particularly hard. Security systems demand early investment to unlock robust recovery strategies elsewhere in the stack.</span></p>
<h3><span style="font-weight: 400;">DNS seeps into everything</span></h3>
<p><span style="font-weight: 400;">Even when you use a custom service discovery system, DNS permeates tooling everywhere. Due to its ubiquity, engineers often forget DNS is a network-accessed system. Because it is so robust, they treat it more like a local resource akin to the filesystem. Learning to view it as the fallible, remote system it is requires rigor and constant recalibration. Strict testing frameworks really help with this, especially in an environment of </span><a href="https://engineering.fb.com/2017/01/17/production-engineering/legacy-support-on-ipv6-only-infra/"><span style="font-weight: 400;">IPv6</span></a><span style="font-weight: 400;">, where 128-bit addresses aren’t easy to remember or write on a sticky note.</span></p>
<p><span style="font-weight: 400;">For select systems that power discovery, we’ve moved to relying on routing protocols to stabilize well-known IP addresses. Across our vast collection of tooling, we’re retrofitting much of it with the ability to accept IP addresses in a pinch.</span></p>
<h3><span style="font-weight: 400;">Caching obscures RPCs</span><span style="font-weight: 400;"> </span></h3>
<p><span style="font-weight: 400;">Years of work have made our systems very difficult to fully break, hardening them with caching and fallbacks that balance trade-offs in freshness vs. availability. While this is great for reliability, this gentle failover behavior makes it challenging to predict how worst-case failure modes will eventually manifest. The first time that service owners see what production would be like without any configuration or service discovery caching, for instance, often occurs in a BellJar environment. For this reason, the team that builds BellJar spends a lot of time ensuring that we’re actually breaking things inside our test environments in the worst way possible.</span></p>
<h3><span style="font-weight: 400;">Even configuration needs configuration</span></h3>
<p><span style="font-weight: 400;">At our scale, everything is configuration-as-code, including the feature knobs and emergency killswitches we deploy for every layer in the stack. We deploy RPC protocol library features and control security ACLs via configuration. Hardware package installations get defined as configuration. Even our configuration distribution systems behave according to configuration settings. The same applies to emergency CLI tooling and fleetwide daemons. When all this configuration is authored and distributed via a common system, it’s worth your time to design robust mechanisms for emergency delivery, automatic fallback to prepackaged values. Additionally, you need to design a diverse set of contingency plans for whenever the configuration that powers your configuration system is broken.</span></p>
<h3><span style="font-weight: 400;">Libraries demand allowlist-style validation</span></h3>
<p><span style="font-weight: 400;">At the pace we move, even low-level services lean on libraries that constantly change in ways no single engineering team can scrutinize line by line. Those libraries can be a vector for unexpected dependency graph creep. With enormous codebases and thousands of services, relying on traditional blocklist-style fault injection to understand how your system interacts with production can be a game of whack-a-mole. In other words, you don’t know what to test if you can’t even enumerate all the services in production.</span></p>
<p><span style="font-weight: 400;">We’ve found unwanted dependencies inserted into our lowest-level systems thanks to simple drop-in Python2 -&gt; Python3 upgrades, which have made recovery tooling dependent on esoteric services. Authorization checks have introduced hard dependencies between emergency utilities and our web front end. Ubiquitous logging and tracing libraries have found themselves unexpectedly promoted to SIGABRT-ing blockers to our container tooling. Thanks to allowlist-style tests like those provided by BellJar, we can automatically detect new dependencies on systems we didn’t even know existed.</span></p>
<h2><span style="font-weight: 400;">Asking new questions</span></h2>
<p><span style="font-weight: 400;">We’re still being surprised by what our services can learn through BellJar’s tightly constrained testing environment. And we’re expanding the set of use cases for BellJar.</span></p>
<p><span style="font-weight: 400;">Now, beyond its initial function for verifying the recovery strategies for our services, we also use BellJar to zoom in on other important pieces of code that don’t look like containers or network services at all.</span></p>
<p><span style="font-weight: 400;">Common libraries like our internal</span><a href="https://engineering.fb.com/2013/11/21/core-data/under-the-hood-building-and-open-sourcing-rocksdb/"> <span style="font-weight: 400;">RocksDB</span></a><span style="font-weight: 400;"> build and</span><a href="https://engineering.fb.com/2019/08/02/core-data/systems-scale/"> <span style="font-weight: 400;">distributed tracing</span></a><span style="font-weight: 400;"> framework now use BellJar to assert that they can gracefully initialize and satisfy requests in environments where none of Meta’s upstream systems are online. Tests like these protect against dependency creep that could find these libraries blocking recoveries for scores of infra systems.</span></p>
<p><span style="font-weight: 400;">We’ve also expanded BellJar to vet the sidecar binaries we run on millions of machines to support basic data distribution, host management, debugging, and traffic shaping. With extremely restricted allowlists, we continually evaluate new daemon builds. Each version must demonstrate it can auto-recover from failure, even when the upstream control planes they typically listen to go dark.</span></p>
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-facebook"><a rel="nofollow noopener noreferrer"
data-shared="sharing-facebook-18735"
class="share-facebook sd-button share-icon"
href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/?share=facebook"
target="_blank"
aria-labelledby="sharing-facebook-18735"
>
<span id="sharing-facebook-18735" hidden>Share on Facebook (Opens in new window)</span>
<span>Facebook</span>
</a></li><li class="share-threads"><a rel="nofollow noopener noreferrer"
data-shared="sharing-threads-18735"
class="share-threads sd-button share-icon"
href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/?share=threads"
target="_blank"
aria-labelledby="sharing-threads-18735"
>
<span id="sharing-threads-18735" hidden>Share on Threads (Opens in new window)</span>
<span>Threads</span>
</a></li><li class="share-jetpack-whatsapp"><a rel="nofollow noopener noreferrer"
data-shared="sharing-whatsapp-18735"
class="share-jetpack-whatsapp sd-button share-icon"
href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/?share=jetpack-whatsapp"
target="_blank"
aria-labelledby="sharing-whatsapp-18735"
>
<span id="sharing-whatsapp-18735" hidden>Share on WhatsApp (Opens in new window)</span>
<span>WhatsApp</span>
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
data-shared="sharing-linkedin-18735"
class="share-linkedin sd-button share-icon"
href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/?share=linkedin"
target="_blank"
aria-labelledby="sharing-linkedin-18735"
>
<span id="sharing-linkedin-18735" hidden>Share on LinkedIn (Opens in new window)</span>
<span>LinkedIn</span>
</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
data-shared="sharing-reddit-18735"
class="share-reddit sd-button share-icon"
href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/?share=reddit"
target="_blank"
aria-labelledby="sharing-reddit-18735"
>
<span id="sharing-reddit-18735" hidden>Share on Reddit (Opens in new window)</span>
<span>Reddit</span>
</a></li><li class="share-x"><a rel="nofollow noopener noreferrer"
data-shared="sharing-x-18735"
class="share-x sd-button share-icon"
href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/?share=x"
target="_blank"
aria-labelledby="sharing-x-18735"
>
<span id="sharing-x-18735" hidden>Share on X (Opens in new window)</span>
<span>X</span>
</a></li><li class="share-bluesky"><a rel="nofollow noopener noreferrer"
data-shared="sharing-bluesky-18735"
class="share-bluesky sd-button share-icon"
href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/?share=bluesky"
target="_blank"
aria-labelledby="sharing-bluesky-18735"
>
<span id="sharing-bluesky-18735" hidden>Share on Bluesky (Opens in new window)</span>
<span>Bluesky</span>
</a></li><li class="share-mastodon"><a rel="nofollow noopener noreferrer"
data-shared="sharing-mastodon-18735"
class="share-mastodon sd-button share-icon"
href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/?share=mastodon"
target="_blank"
aria-labelledby="sharing-mastodon-18735"
>
<span id="sharing-mastodon-18735" hidden>Share on Mastodon (Opens in new window)</span>
<span>Mastodon</span>
</a></li><li class="share-custom share-custom-hackernews"><a rel="nofollow noopener noreferrer"
data-shared="sharing-custom-18735"
class="share-custom share-custom-hackernews sd-button share-icon"
href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/?share=custom-1699562127"
target="_blank"
aria-labelledby="sharing-custom-18735"
>
<span id="sharing-custom-18735" hidden>Share on Hacker News (Opens in new window)</span>
<span style="background-image:url(&quot;https://news.ycombinator.com/y18.svg&quot;);">Hacker News</span>
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
data-shared="sharing-email-18735"
class="share-email sd-button share-icon"
href="mailto:?subject=%5BShared%20Post%5D%20BellJar%3A%20A%20new%20framework%20for%20testing%20system%20recoverability%20at%20scale&#038;body=https%3A%2F%2Fengineering.fb.com%2F2022%2F05%2F05%2Fdeveloper-tools%2Fbelljar%2F&#038;share=email"
target="_blank"
aria-labelledby="sharing-email-18735"
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you&#039;re having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="c26907d7a9" data-email-share-track-url="https://engineering.fb.com/2022/05/05/developer-tools/belljar/?share=email">
<span id="sharing-email-18735" hidden>Email a link to a friend (Opens in new window)</span>
<span>Email</span>
</a></li><li class="share-end"></li></ul></div></div></div>
</div><!-- .entry-content -->
</article><!-- #post-## -->
<footer class="entry-footer">
</footer><!-- .entry-footer -->
<div class="fb-like"
data-href="https://engineering.fb.com/2022/05/05/developer-tools/belljar/"
data-width=""
data-layout="standard"
data-action="like"
data-size="small"
data-share="true">
</div>
<div class="related-articles">
<div class="related-articles__header">
<h3 class="related-articles__header__headline">Read More in DevInfra</h3>
<a
aria-label="View all articles in this DevInfra"
class="related-articles__header__link"
href="https://engineering.fb.com/category/developer-tools/"
>
View All </a>
</div>
<div class="article-grids related-articles__grid">
<div class="row">
<div class="article-grid col-12 col-md-4">
<article class="post post-18735 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure category-developer-tools fb_content_type-article" id="post-24394">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/Meta-MTIA-300-Hero-final.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Meta MTIA 300 - Hero - final" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-08-24T10:45:52-07:00">
AUG 24, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/mtia-300-meta-training-chip-built-in-nics/" rel="bookmark" aria-label="Read More about MTIA 300: Meta&#8217;s First Training Chip with Built-in NICs and Communication-Offloading Engines">
MTIA 300: Meta&#8217;s First Training Chip with Built-in NICs and Communication-Offloading Engines </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18735 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure category-developer-tools fb_content_type-article" id="post-24128">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/10-Years-Meta-Python-HERO-large.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-30T09:00:46-07:00">
JUN 30, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/30/open-source/10-years-of-metas-commitment-to-python/" rel="bookmark" aria-label="Read More about 10 Years of Meta’s Commitment to Python">
10 Years of Meta’s Commitment to Python </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18735 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure category-developer-tools fb_content_type-article" id="post-23898">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/capacity_efficiency_hero_white_option_5_1775676974.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="capacity_efficiency_hero_white_option_5_1775676974" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-04-16T09:00:34-07:00">
APR 16, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/04/16/developer-tools/capacity-efficiency-at-meta-how-unified-ai-agents-optimize-performance-at-hyperscale/" rel="bookmark" aria-label="Read More about Capacity Efficiency at Meta: How Unified AI Agents Optimize Performance at Hyperscale">
Capacity Efficiency at Meta: How Unified AI Agents Optimize Performance at Hyperscale </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18735 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure category-developer-tools fb_content_type-article" id="post-23856">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Escaping-the-Fork-Hero-2.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Escaping the Fork - Hero 2" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-04-09T09:00:34-07:00">
APR 9, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/04/09/developer-tools/escaping-the-fork-how-meta-modernized-webrtc-across-50-use-cases/" rel="bookmark" aria-label="Read More about Escaping the Fork: How Meta Modernized WebRTC Across 50+ Use Cases">
Escaping the Fork: How Meta Modernized WebRTC Across 50+ Use Cases </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18735 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure category-developer-tools fb_content_type-article" id="post-23871">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Meta-Tech-Podcast-episode-84-Configuration-Safety-at-Scale.webp?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-04-08T11:25:26-07:00">
APR 8, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/04/08/security/trust-but-canary-configuration-safety-at-scale-meta-tech-podcast/" rel="bookmark" aria-label="Read More about Trust But Canary: Configuration Safety at Scale">
Trust But Canary: Configuration Safety at Scale </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18735 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure category-developer-tools fb_content_type-article" id="post-23840">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Compass-Not-Enclycopedia-Hero.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-04-06T09:00:50-07:00">
APR 6, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/04/06/developer-tools/how-meta-used-ai-to-map-tribal-knowledge-in-large-scale-data-pipelines/" rel="bookmark" aria-label="Read More about How Meta Used AI to Map Tribal Knowledge in Large-Scale Data Pipelines">
How Meta Used AI to Map Tribal Knowledge in Large-Scale Data Pipelines </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
</div><!-- /.row -->
</div><!--/.article-grids -->
</div><!-- /.related-articles -->
<!-- include the carousel nav here. -->
</main><!-- #main -->
</div><!-- #primary -->
<!-- Do the right sidebar check. -->
<div class="col-md-4 widget-area" id="right-sidebar" role="complementary">
<aside id="fb_related_post_widget-4" class="widget widget_fb_related_post_widget"><h3 class="widget-title">Related Posts</h3><hr>
<div class="aside-related-projects"><div class="article-grid-redesign">
<article class="post-16635 post type-post status-publish has-post-thumbnail hentry" id="post-16635">
<a href="https://engineering.fb.com/2020/09/08/data-center-engineering/fault-tolerance-through-optimal-workload-placement/" rel="bookmark" aria-label="Read More about Fault tolerance through optimal workload placement">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2020/09/WorkloadPlacement_hero_v1.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2020/09/WorkloadPlacement_hero_v1.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2020/09/WorkloadPlacement_hero_v1.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Sep 08, 2020">Sep 08, 2020</time></span>
<h4 class="entry-title">Fault tolerance through optimal workload placement</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div><div class="article-grid-redesign">
<article class="post-18420 post type-post status-publish has-post-thumbnail hentry" id="post-18420">
<a href="https://engineering.fb.com/2021/12/16/data-center-engineering/power-loss-siren/" rel="bookmark" aria-label="Read More about Power Loss Siren: Making Meta resilient to power loss events">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2021/12/Eng-Blog-Self-Serve-Hero-Images-DATACENTERS-501-Teale.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2021/12/Eng-Blog-Self-Serve-Hero-Images-DATACENTERS-501-Teale.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2021/12/Eng-Blog-Self-Serve-Hero-Images-DATACENTERS-501-Teale.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Dec 16, 2021">Dec 16, 2021</time></span>
<h4 class="entry-title">Power Loss Siren: Making Meta resilient to power loss events</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div><div class="article-grid-redesign">
<article class="post-5910 post type-post status-publish has-post-thumbnail hentry" id="post-5910">
<a href="https://engineering.fb.com/2013/11/21/core-infra/under-the-hood-building-and-open-sourcing-rocksdb/" rel="bookmark" aria-label="Read More about Under the Hood: Building and open-sourcing RocksDB">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2013/11/rocksdb.png?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2013/11/rocksdb.png?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2013/11/rocksdb.png?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Nov 21, 2013">Nov 21, 2013</time></span>
<h4 class="entry-title">Under the Hood: Building and open-sourcing RocksDB</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div></div></aside><aside id="wpb_widget-3" class="widget widget_wpb_widget"><h3 class="widget-title">Related Positions</h3><hr><ul class="available-positions-list"> </ul>
<a href="https://www.metacareers.com">
<div class="btn available-positions-btn">
See All Jobs </div>
</a>
</aside>
</div><!-- #secondary -->
<!-- <php get_template_part('content-single-side-bar'); ?> -->
<!-- Do the right sidebar check. -->
</div><!-- Container end -->
</div><!-- Wrapper end -->
</div>
<!-- ******************* The Footer Full-width Widget Area ******************* -->
<div class="wrapper wrapper-community-outer">
<div class="wrapper-community">
<div class="community" tabindex="-1">
<div class="row">
<!-- community section is generated through WordPress widget -->
<div id="wpb_widget-2" class="footer-widget widget_wpb_widget widget-count-3 col-md-4"><h3 class="widget-title">Available Positions</h3><hr><ul class="available-positions-list"> </ul>
<a href="https://www.metacareers.com">
<div class="btn available-positions-btn">
See All Jobs </div>
</a>
</div><!-- .footer-widget --><div id="custom_html-2" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Technology at Meta</h3><div class="textwidget custom-html-widget"><ul class="stay-connected-lists">
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-fb-engineering" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Engineering at Meta - X
</div>
<a onclick='return twitterFollow("Meta_Engineers")'>
<button class="btn stay-connected-btn">
Follow
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-AI" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
AI at Meta
</div>
<a href="https://ai.meta.com/blog/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta Quest Blog
</div>
<a href="https://www.meta.com/blog/quest/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta for Developers
</div>
<a href="https://developers.facebook.com/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-bug-bounty" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta Bug Bounty
</div>
<a href="https://bugbounty.meta.com/">
<button class="btn stay-connected-btn">
Learn more
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/rss.png" alt="footer-rss" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
RSS
</div>
<a href="https://code.facebook.com/posts/rss/">
<button class="btn stay-connected-btn">
Subscribe
</button>
</a>
</div>
</li>
</ul>
</div></div><!-- .footer-widget --><div id="custom_html-3" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Open Source</h3><div class="textwidget custom-html-widget"><p class="opensource-desc">
Meta believes in building community through open source technology. Explore our latest projects in Artificial Intelligence, Data Infrastructure, Development Tools, Front End, Languages, Platforms, Security, Virtual Reality, and more.
</p>
<ul class="opensource-cats">
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/android.png" alt="android" loading="lazy" width="50" height="59">
<div class="opensource-cap">
ANDROID
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/ios.png" alt="ios" loading="lazy" width="48" height="58">
<div class="opensource-cap">
iOS
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/web.png" alt="web" loading="lazy" width="55" height="49">
<div class="opensource-cap">
WEB
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/backend.png" alt="backend" loading="lazy" width="56" height="51">
<div class="opensource-cap">
BACKEND
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/hardware.png" alt="hardware" loading="lazy" width="55" height="44">
<div class="opensource-cap">
HARDWARE
</div>
</li>
</ul>
<button class="btn opensource-btn" onclick="window.location.href='https://opensource.fb.com/'">
Learn More
</button>
</div></div><!-- .footer-widget -->
</div>
</div>
</div>
</div><!-- #wrapper-footer-full -->
<div class="wrapper-footer" id="wrapper-footer">
<div class="inner-container">
<div class="footer">
<div class="row">
<div class="col-md-12">
<footer class="meta-footer" id="meta-footer">
<div class="site-info row justify-content-between">
<div class="col-md-5 meta-logo-container">
<a href="https://about.facebook.com/">
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/meta_logo_full.svg" width="89" height="18" alt="Meta" />
</a>
<div id="text-2" class="widget widget_text"> <div class="textwidget"><p>Engineering at Meta is a technical news resource for engineers interested in how we solve large-scale technical challenges at Meta.</p>
</div>
</div> </div>
<div class="col-md-6 footer-meta-links-container">
<ul class="meta-footer-links">
<li class="footer-link"><a href="https://engineering.fb.com">Home</a></li><li class="footer-link"><a href="https://about.meta.com/">Company Info</a></li><li class="footer-link"><a href="https://www.metacareers.com/?ref=engineering.fb.com">Careers</a></li> </ul>
</div>
</div>
</footer>
</div>
</div>
</div>
</div>
<div class="footer">
<div class="row">
<div class="col-md-12">
<footer class="site-footer" id="colophon">
<div class="inner-container">
<div class="site-info row justify-content-between">
<div class="col-md-4 copyright-container">
<div class="copyright"> © 2026 Meta</div>
</div>
<!-- static footer -->
<div class="col-md-7 footer-links-container">
<ul class="footer-links">
<li class="footer-link"><a href="https://www.facebook.com/policies">Terms</a></li><li class="footer-link"><a href="https://www.facebook.com/privacy/policy">Privacy</a></li><li class="footer-link"><a href="/privacy">Cookies</a></li><li class="footer-link"><a href="https://www.facebook.com/help">Help</a></li> </ul>
</div>
</div>
</div><!-- .site-info -->
</footer><!-- #colophon -->
</div><!--col end -->
</div><!-- row end -->
</div><!-- //.inner-container -->
</div><!-- wrapper end -->
</div><!-- #page we need this extra closing tag here -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/code-fb-com/*","/wp-content/themes/understrap/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<div class="gdprconsent-container" id="GDPRConsentBar">
<div class="gdprconsent-wrapper">
<div class="gdprconsent-content">
<p>To help personalize content, tailor and measure ads and provide a safer experience, we use cookies. By clicking or navigating the site, you agree to allow our collection of information on and off Facebook through cookies. Learn more, including about available controls: <a href="/privacy">Cookie Policy</a></p>
<div class="gdprconsent-button">
<button type="button" data-gdpr-accept>Accept</button>
</div>
</div>
</div>
</div>
<!-- Twitter universal website tag code -->
<script>
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
// Insert Twitter Pixel ID and Standard Event data below
twq('init','o31u0');
twq('track','PageView');
});
}
</script>
<!-- End Twitter universal website tag code -->
<script type="text/javascript">
window.WPCOM_sharing_counts = {"https://engineering.fb.com/2022/05/05/developer-tools/belljar/":18735};
</script>
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KZRLGZ8" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=660742068067366&ev=PageView&noscript=1"/></noscript>
<script id="child-understrap-scripts-js-extra">
var loadmore_params = {"restfulURL":"https://engineering.fb.com/wp-json/fb/v1/loadmore","searchText":"","posts":"{\"page\":0,\"year\":2022,\"monthnum\":5,\"day\":5,\"name\":\"belljar\",\"category_name\":\"developer-tools\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","post_type":"post"};
var search_params = {"ajaxurl":"https://engineering.fb.com/wp-admin/admin-ajax.php","posts":"{\"page\":0,\"year\":2022,\"monthnum\":5,\"day\":5,\"name\":\"belljar\",\"category_name\":\"developer-tools\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","article_filter":"2f23ad5ce3"};
var more_articles = {"next_article":{"label":"Next","permalink":"https://engineering.fb.com/2022/05/09/android/language-packs/","title":"Language packs: Meta\u2019s mobile localization solution","thumbnail":["https://engineering.fb.com/wp-content/uploads/2022/05/LanguagePacksHero.jpg?w=580&h=326&crop=1",580,326,true]},"prev_article":{"label":"Prev","permalink":"https://engineering.fb.com/2022/05/04/data-infrastructure/delta/","title":"Delta: A highly available, strongly consistent storage service using chain replication","thumbnail":["https://engineering.fb.com/wp-content/uploads/2022/05/Delta-Title-Slide2.jpg?w=580&h=326&crop=1",580,326,true]}};
//# sourceURL=child-understrap-scripts-js-extra
</script>
<script id="jetpack-facebook-embed-js-extra">
var jpfbembed = {"appid":"249643311490","locale":"en_US"};
//# sourceURL=jetpack-facebook-embed-js-extra
</script>
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJx9jUEOwiAQAD/kshJjqwfjU0xZtnZpF0iB+H2J8eTB62Qmg68MlGLlWLEurFyQkmeYXceKXkrF0Nkim4ePYFSiCeWA/8se5V2K/qraIG/tKbFg4JonWsEOxuJDIqFrfYPzROxSWoHVsf/+7nqz4+V6HM/DyYY3/BNA4A==" ></script><script id="sharing-js-js-extra">
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":""};
//# sourceURL=sharing-js-js-extra
</script>
<script id="sharing-js-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/_inc/build/sharedaddy/sharing.min.js?ver=16.1.3"></script>
<script id="sharing-js-js-after">
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-facebook' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-threads' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-threads' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomthreads', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-linkedin' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-x' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-x' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomx', 'menubar=1,resizable=1,width=600,height=350' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-bluesky' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-bluesky' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcombluesky', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-mastodon' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-mastodon' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcommastodon', 'menubar=1,resizable=1,width=460,height=400' );
return false;
}
} );
} )();
//# sourceURL=sharing-js-js-after
</script>
<script id="video-analytics-js-after">
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
let youtube = document.querySelectorAll('[src*="youtube-nocookie"]');
for (let i = 0; i < youtube.length; i++) {
youtube[i].src = youtube[i].src.replace('youtube-nocookie.com', 'youtube.com');
}
let vimeo = document.querySelectorAll('[src*="vimeo"]');
for (let i = 0; i < vimeo.length; i++) {
vimeo[i].src = vimeo[i].src.replace('&dnt=true', '');
}
});
}
//# sourceURL=video-analytics-js-after
</script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://engineering.fb.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://engineering.fb.com/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>