Files
nexus/sreweekly/articles/306/04-foqs-making-a-distributed-priority-queue-disaster-ready.html
2026-09-12 17:23:01 +08:00

1198 lines
106 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-title" content="Engineering at Meta - Engineering at Meta Blog">
<meta property="fb:app_id" content="1425766027653270" />
<link rel="profile" href="http://gmpg.org/xfn/11">
<link rel="pingback" href="https://engineering.fb.com/xmlrpc.php">
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
<script async src="https://engineering.fb.com/wp-content/themes/code-fb-com/dist/js/optimize.min.js"></script>
<!-- This site is optimized with the Yoast SEO Premium plugin v19.3 (Yoast SEO v27.3) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
<title>FOQS: Making a distributed priority queue disaster-ready - Engineering at Meta</title>
<meta name="description" content="FOQS has evolved from a regional to a global deployment for highly available data, even through large-scale disaster scenarios." />
<link rel="canonical" href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/" />
<meta name="twitter:label1" content="Written by" />
<meta name="twitter:data1" content="Dillon George, Jasmit Kaur Saluja, changigi649" />
<meta name="twitter:label2" content="Est. reading time" />
<meta name="twitter:data2" content="11 minutes" />
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/#article","isPartOf":{"@id":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/"},"author":[{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/f4ab150bf66c8508b2066fb324792f44"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/c3dee16bd44816d00ee8f683459ca8ef"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/7167c692371a46d5bb5879bbd971c6c0"}],"headline":"FOQS: Making a distributed priority queue disaster-ready","datePublished":"2022-01-18T17:00:21+00:00","dateModified":"2022-01-18T19:00:36+00:00","mainEntityOfPage":{"@id":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/"},"wordCount":2149,"publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"image":{"@id":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/01\/Hero.jpeg","articleSection":["Production Engineering"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/","url":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/","name":"FOQS: Making a distributed priority queue disaster-ready - Engineering at Meta","isPartOf":{"@id":"https:\/\/engineering.fb.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/#primaryimage"},"image":{"@id":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/01\/Hero.jpeg","datePublished":"2022-01-18T17:00:21+00:00","dateModified":"2022-01-18T19:00:36+00:00","description":"FOQS has evolved from a regional to a global deployment for highly available data, even through large-scale disaster scenarios.","breadcrumb":{"@id":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/#primaryimage","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/01\/Hero.jpeg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/01\/Hero.jpeg","width":1920,"height":1080,"caption":"FOQS: Making a distributed priority queue disaster-ready"},{"@type":"BreadcrumbList","@id":"https:\/\/engineering.fb.com\/2022\/01\/18\/production-engineering\/foqs-disaster-ready\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/engineering.fb.com\/"},{"@type":"ListItem","position":2,"name":"FOQS: Making a distributed priority queue disaster-ready"}]},{"@type":"WebSite","@id":"https:\/\/engineering.fb.com\/#website","url":"https:\/\/engineering.fb.com\/","name":"Engineering at Meta","description":"Engineering at Meta Blog","publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/engineering.fb.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/engineering.fb.com\/#organization","name":"Meta","url":"https:\/\/engineering.fb.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","width":29011,"height":12501,"caption":"Meta"},"image":{"@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/Engineering\/","https:\/\/x.com\/fb_engineering"]},[],{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/f4ab150bf66c8508b2066fb324792f44","name":"Dillon George","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Dillon George"},"url":"https:\/\/engineering.fb.com\/author\/dillon-george\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/c3dee16bd44816d00ee8f683459ca8ef","name":"Jasmit Kaur Saluja","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Jasmit Kaur Saluja"},"url":"https:\/\/engineering.fb.com\/author\/jasmit-kaur-saluja\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/7167c692371a46d5bb5879bbd971c6c0","name":"changigi649","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/a090692c0af9b5ed6f8071e0a05e3147f0ba61b52a51c0671d073d9adb4e198e?s=96&d=mm&r=g8fd1e2b34e75c09c4aea578e73cbde79","url":"https:\/\/secure.gravatar.com\/avatar\/a090692c0af9b5ed6f8071e0a05e3147f0ba61b52a51c0671d073d9adb4e198e?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/a090692c0af9b5ed6f8071e0a05e3147f0ba61b52a51c0671d073d9adb4e198e?s=96&d=mm&r=g","caption":"changigi649"},"url":"https:\/\/engineering.fb.com\/author\/changigi649\/"}]}</script>
<!-- / Yoast SEO Premium plugin. -->
<link rel='dns-prefetch' href='//secure.gravatar.com' />
<link rel='dns-prefetch' href='//v0.wordpress.com' />
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta &raquo; Feed" href="https://engineering.fb.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta &raquo; Comments Feed" href="https://engineering.fb.com/comments/feed/" />
<script async src="https://www.googleoptimize.com/optimize.js?id=OPT-TPNDQC5" onerror="dataLayer.hide.end && dataLayer.hide.end()"></script>
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2022%2F01%2F18%2Fproduction-engineering%2Ffoqs-disaster-ready%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2022%2F01%2F18%2Fproduction-engineering%2Ffoqs-disaster-ready%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='all-css-12' href='https://engineering.fb.com/_static/??-eJyNjUsOwjAMRC9EaiqghQXiKFVqhzSQn+pEiNuTFJBAbNjZTzNv4BYFBp+UTxBt1sYzFKQpzpVz5WQ4QSUv0CDzCj6KaVJOMWAgJc5jwe7ZKTnAyVgSS6Jxxv90XRbv3YtKUeJVtF3TgguUbZHyJGdFkui+nMbrvxWD8QhjLvvAAY20wgYd+OupspM7tv3+sO533Wb7AImQY4U=' type='text/css' media='all' />
<script id="gdprconsent-js" src="https://engineering.fb.com/wp-content/plugins/wp-gdpr-consent/dist/gdprconsent.js?ver=2.0.0"></script>
<script id="gdprconsent-js-after">
if (window.gdprSafeTrack) {
window.gdprSafeTrack('https://www.googletagmanager.com/gtag/js?id=391063774');
window.gdprSafeTrack(function() {
var google_analytics_g4 = decodeURIComponent( '391063774' );
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', google_analytics_g4);
});
}
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
var google_tag_manager = decodeURIComponent( 'GTM-KZRLGZ8' );
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer',google_tag_manager);
});
}
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
var facebook_pixel = decodeURIComponent( '660742068067366' );
!function(f,b,e,v,n,t,s)
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
n.queue=[];t=b.createElement(e);t.async=!0;
t.src=v;s=b.getElementsByTagName(e)[0];
s.parentNode.insertBefore(t,s)}(window, document,'script',
'https://connect.facebook.net/en_US/fbevents.js');
fbq('init', facebook_pixel);
fbq('track', 'PageView');
});
}
//# sourceURL=gdprconsent-js-after
</script>
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJzTLy/QzcxLzilNSS3WzwKiwtLUokoopZebmaeXVayjj0+Rbm5melFiSSpUsX2uraG5hbmZgYmloUkWAK+3Iic=" ></script><script type="text/javascript" src="https://engineering.fb.com/wp-content/themes/understrap/js/popper.min.js?m=1773259670g" ></script><link rel="https://api.w.org/" href="https://engineering.fb.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.fb.com/wp-json/wp/v2/posts/18459" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.fb.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress 7.1" />
<link rel='shortlink' href='https://wp.me/pa0Lhq-4NJ' />
<link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/Optimistic_Display_W_Md.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="FOQS: Making a distributed priority queue disaster-ready" />
<meta property="og:url" content="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/" />
<meta property="og:description" content="Facebook Ordered Queueing Service (FOQS) is a fully managed, distributed priority queueing service used for reliable message delivery among many services. FOQS has evolved from a regional deploymen…" />
<meta property="article:published_time" content="2022-01-18T17:00:21+00:00" />
<meta property="article:modified_time" content="2022-01-18T19:00:36+00:00" />
<meta property="og:site_name" content="Engineering at Meta" />
<meta property="og:image" content="https://engineering.fb.com/wp-content/uploads/2022/01/Hero.jpeg" />
<meta property="og:image:width" content="1920" />
<meta property="og:image:height" content="1080" />
<meta property="og:image:alt" content="FOQS: Making a distributed priority queue disaster-ready" />
<meta property="og:locale" content="en_US" />
<meta name="twitter:text:title" content="FOQS: Making a distributed priority queue disaster-ready" />
<meta name="twitter:image" content="https://engineering.fb.com/wp-content/uploads/2022/01/Hero.jpeg?w=640" />
<meta name="twitter:image:alt" content="FOQS: Making a distributed priority queue disaster-ready" />
<meta name="twitter:card" content="summary_large_image" />
<!-- End Jetpack Open Graph Tags -->
<style id="wp-custom-css">
.table-of-contents {
position: sticky;
top: 20px;
background: #fff;
padding: 1em;
border: 1px solid #ddd;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
max-width: 250px;
}
</style>
<link rel="shortcut icon" href="https://engineering.fb.com/wp-content/themes/code-fb-com/favicon.ico" />
</head>
<body class="wp-singular post-template-default single single-post postid-18459 single-format-standard wp-theme-understrap wp-child-theme-code-fb-com group-blog">
<div class="hfeed site" id="page">
<!-- ******************* The Navbar Area ******************* -->
<div class="wrapper-fluid wrapper-navbar" id="wrapper-navbar">
<a class="skip-link screen-reader-text sr-only" href="#content">
Skip to content </a>
<nav class="navbar navbar-expand-std">
<div class="nav-container">
<!-- Your site title as branding in the menu -->
<div class="nav-top-row">
<h1 class="navbar-brand mb-0">
<a rel="home" href="https://engineering.fb.com/" title="Engineering at Meta">
<img class="navbar-brand-logo" src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/logo-meta.svg" class="navbar-brand-logo" alt="Engineering at Meta" width="215" height="40" loading="lazy">
</a>
</h1>
<!-- end custom logo -->
<label for="search" class="screen-reader-text">Search this site</label>
<input type="text" id="search" name="search" class="form-control-lg navbar-search" autocomplete="off">
<button class="navbar-toggler navbar-searchbar-toggler"
type="button"
data-toggle="collapse"
data-target="#searchbarDropdown"
aria-controls="searchbarDropdown"
aria-expanded="false"
aria-label="Toggle navigation">
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/icon-search.svg" class="navbar-mobile-search-icon" loading="lazy" width="24" height="25" />
</button>
<!-- desktop language menu (mobile language menu is implemented via jQuery)-->
<div class="notranslate nice-select navbar-language form-control">
</div>
<button class="navbar-toggler collapsed navbar-menu-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<div class="navbar-border"></div>
<!-- The WordPress Menu goes here -->
<div id="navbarNavDropdown" class="collapse navbar-collapse"><ul id="main-menu" class="navbar-nav"><li id="menu-item-60" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-60 dropdown"><a title="Open Source" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Open Source <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-12330" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-12330"><a title="Open Source" href="https://engineering.fb.com/category/open-source/" class="nav-link">Open Source</a></li>
<li id="menu-item-12331" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-12331"><a title="Meta Open Source" href="https://opensource.fb.com" class="nav-link">Meta Open Source</a></li>
</ul>
</li>
<li id="menu-item-2686" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2686 dropdown"><a title="Platforms" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Platforms <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-65" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-65"><a title="Android" href="https://engineering.fb.com/category/android/" class="nav-link">Android</a></li>
<li id="menu-item-66" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-66"><a title="iOS" href="https://engineering.fb.com/category/ios/" class="nav-link">iOS</a></li>
<li id="menu-item-67" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-67"><a title="Web" href="https://engineering.fb.com/category/web/" class="nav-link">Web</a></li>
</ul>
</li>
<li id="menu-item-2682" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2682 dropdown"><a title="Infrastructure Systems" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Infrastructure Systems <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-218" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-218"><a title="Core Infra" href="https://engineering.fb.com/category/core-infra/" class="nav-link">Core Infra</a></li>
<li id="menu-item-219" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-219"><a title="Data Infrastructure" href="https://engineering.fb.com/category/data-infrastructure/" class="nav-link">Data Infrastructure</a></li>
<li id="menu-item-220" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-220"><a title="DevInfra" href="https://engineering.fb.com/category/developer-tools/" class="nav-link">DevInfra</a></li>
<li id="menu-item-226" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-226"><a title="Production Engineering" href="https://engineering.fb.com/category/production-engineering/" class="nav-link">Production Engineering</a></li>
<li id="menu-item-227" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-227"><a title="Security &amp; Privacy" href="https://engineering.fb.com/category/security/" class="nav-link">Security &amp; Privacy</a></li>
<li id="menu-item-18607" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18607"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/systems-infrastructure/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2683" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2683 dropdown"><a title="Physical Infrastructure" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Physical Infrastructure <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-215" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-215"><a title="Connectivity" href="https://engineering.fb.com/category/connectivity/" class="nav-link">Connectivity</a></li>
<li id="menu-item-216" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-216"><a title="Data Center Engineering" href="https://engineering.fb.com/category/data-center-engineering/" class="nav-link">Data Center Engineering</a></li>
<li id="menu-item-217" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-217"><a title="Networking &amp; Traffic" href="https://engineering.fb.com/category/networking-traffic/" class="nav-link">Networking &amp; Traffic</a></li>
<li id="menu-item-18608" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18608"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/networking-connectivity/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2684" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2684 dropdown"><a title="Video Engineering &amp; AR/VR" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Video Engineering &amp; AR/VR <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-225" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-225"><a title="Video Engineering" href="https://engineering.fb.com/category/video-engineering/" class="nav-link">Video Engineering</a></li>
<li id="menu-item-223" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-223"><a title="Virtual Reality" href="https://engineering.fb.com/category/virtual-reality/" class="nav-link">Virtual Reality</a></li>
<li id="menu-item-13388" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13388"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/augmented-reality-virtual-reality/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2685" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2685 dropdown"><a title="Artificial Intelligence" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Artificial Intelligence <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-222" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-222"><a title="ML Applications" href="https://engineering.fb.com/category/ml-applications/" class="nav-link">ML Applications</a></li>
<li id="menu-item-221" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-221"><a title="AI Research" href="https://engineering.fb.com/category/ai-research/" class="nav-link">AI Research</a></li>
<li id="menu-item-13387" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13387"><a title="Research Publications" href="https://ai.facebook.com/results/?content_types%5B0%5D=publication" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-228" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-228"><a title="Watch Videos" href="/videos" class="nav-link">Watch Videos</a></li>
</ul></div> </div><!-- .container -->
<div class="search-container collapse" id="searchbarDropdown">
<div class="searchbar-search">
<div class="searchbar-container">
<div class="navbar-mobile-search-margin"></div>
<input type="text" name="search" placeholder="Search" class="form-control-lg navbar-mobile-search" autofocus autocomplete="off">
<button class="searchbar-clear">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
</div>
</div>
</nav><!-- .site-navigation -->
</div><!-- .wrapper-navbar end -->
<div class="wrapper" id="single-wrapper">
<div class="single-container row" id="content" tabindex="-1">
<!-- Do the left sidebar check. -->
<div class="content-area" id="primary">
<main class="site-main" id="main">
<article class="post-18459 post type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-18459">
<header class="entry-header">
<div class="entry-meta">
POSTED ON <span class="posted-on entry-date"><time class="published updated" datetime="2022-01-18">JANUARY 18, 2022</time></span> TO <span class="cat-links"><a href="https://engineering.fb.com/category/production-engineering/" rel="category tag">Production Engineering</a></span>
<h1 class="entry-title"><div>FOQS: Making a distributed priority queue disaster-ready</div></h1> </div><!-- .entry-meta -->
</header><!-- .entry-header -->
<figure id="post-feat-image-container">
<img width="1920" height="1080" src="https://engineering.fb.com/wp-content/uploads/2022/01/Hero.jpeg" class="wp-post-image" alt="FOQS: Making a distributed priority queue disaster-ready" title="Hero" /> </figure>
<div class="entry-authors-social">
<div class="entry-authors">
By <a href="https://engineering.fb.com/author/dillon-george/" title="Posts by Dillon George" class="author url fn" rel="author">Dillon George</a>, <a href="https://engineering.fb.com/author/jasmit-kaur-saluja/" title="Posts by Jasmit Kaur Saluja" class="author url fn" rel="author">Jasmit Kaur Saluja</a> </div>
</div>
<div class="entry-content">
<ul>
<li aria-level="1"><a href="https://engineering.fb.com/2021/02/22/production-engineering/foqs-scaling-a-distributed-priority-queue/"><span style="font-weight: 400;">Facebook Ordered Queueing Service (FOQS)</span></a><span style="font-weight: 400;"> is a fully managed, distributed priority queueing service used for reliable message delivery among many services.</span></li>
</ul>
<ul>
<li aria-level="1"><span style="font-weight: 400;">FOQS has evolved from a regional deployment into a geo-distributed, global deployment to help ensure that data stored within logical queues is highly available, even through large-scale disaster scenarios.</span></li>
</ul>
<ul>
<li aria-level="1"><span style="font-weight: 400;">Migrating to a global architecture required architectural modifications to support routing and care to complete without interrupting service availability for FOQS customers.</span></li>
</ul>
<p><span style="font-weight: 400;"><a href="https://engineering.fb.com/2021/02/22/production-engineering/foqs-scaling-a-distributed-priority-queue/">FOQS</a> is the priority queuing service that ensures reliable message delivery among many of Meta’s apps and services. For example, it is the platform for delivering messages that people leave on Facebook to let friends and family know they are safe in situations like an earthquake. Given the important role that FOQS plays, it must be resilient to large-scale disasters such as the complete failure of an entire data center or region, or planned outages when maintenance is being performed on a data center.</span></p>
<p><span style="font-weight: 400;">Since its inception as a regional installation with one main customer, FOQS has evolved into a global installation. After Hurricane Florence in 2018, we realized that to be truly disaster-ready, we needed to migrate to a global architecture with higher availability such that data in the queue is still globally accessible during disaster events. Along the way, we solved the operational challenges associated with migrating tenants to global installations and completed the migration in 2021 at incredible scale, incurring zero downtime for internal clients.</span></p>
<div class="fb-video" data-allowfullscreen="true" data-href="https://www.facebook.com/atscaleevents/videos/446588570140162" style="background-color: #fff; display: inline-block;"></div>
<h2><span style="font-weight: 400;">Regional installations</span></h2>
<p><span style="font-weight: 400;">FOQS was originally developed in 2018 out of scaling necessity for Meta’s </span><a href="https://engineering.fb.com/2020/08/17/production-engineering/async/"><span style="font-weight: 400;">Asynchronous computing platform (aka Async)</span></a><span style="font-weight: 400;">. With our first iteration, the system was built on top of a three-region, </span><a href="https://dev.mysql.com/doc/refman/5.7/en/replication-semisync.html"><span style="font-weight: 400;">semi-synchronous replicated MySQL topology</span></a><span style="font-weight: 400;"> with queue nodes in region X being completely isolated from queue nodes in region Y. </span></p>
<figure id="attachment_18462" aria-describedby="caption-attachment-18462" style="width: 1920px" class="wp-caption alignnone"><img decoding="async" class="wp-image-18462 size-full" src="https://engineering.fb.com/wp-content/uploads/2022/01/Topology.jpeg" alt="A visualization of the MySQL replication topology used by FOQS. Notice the primary with two secondaries in different regions." width="1920" height="1080" srcset="https://engineering.fb.com/wp-content/uploads/2022/01/Topology.jpeg 1920w, https://engineering.fb.com/wp-content/uploads/2022/01/Topology.jpeg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2022/01/Topology.jpeg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2022/01/Topology.jpeg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2022/01/Topology.jpeg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2022/01/Topology.jpeg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2022/01/Topology.jpeg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2022/01/Topology.jpeg?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-18462" class="wp-caption-text">A visualization of the MySQL replication topology used by FOQS. Notice the primary with two secondaries in different regions.</figcaption></figure>
<p><span style="font-weight: 400;">The nodes in each region manage MySQL replicas with the primary typically within the same region. Clients such as Async would use the FOQS client to talk to queue service in the physical region of their choosing, which then stores items in randomly selected MySQL shards for processing.</span></p>
<figure id="attachment_18463" aria-describedby="caption-attachment-18463" style="width: 1920px" class="wp-caption alignnone"><img decoding="async" class="size-full wp-image-18463" src="https://engineering.fb.com/wp-content/uploads/2022/01/Regional_installations.jpeg" alt="A high-level overview of a regional installation. Notice each physical region has its own separate tier with queue nodes managing MySQL shards within the same region." width="1920" height="1080" srcset="https://engineering.fb.com/wp-content/uploads/2022/01/Regional_installations.jpeg 1920w, https://engineering.fb.com/wp-content/uploads/2022/01/Regional_installations.jpeg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2022/01/Regional_installations.jpeg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2022/01/Regional_installations.jpeg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2022/01/Regional_installations.jpeg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2022/01/Regional_installations.jpeg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2022/01/Regional_installations.jpeg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2022/01/Regional_installations.jpeg?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-18463" class="wp-caption-text">A high-level overview of a regional installation. Notice each physical region has its own separate tier with queue nodes managing MySQL shards within the same region.</figcaption></figure>
<h3><span style="font-weight: 400;">Benefits of regional installations</span></h3>
<p><span style="font-weight: 400;">Customers of FOQS bear the responsibility of selecting a physical region to serve their requests. Considering that the complexities of regional outages and global load balancing are pushed to clients, FOQS’s regional architecture remains simple. Regional installations are also horizontally scalable, as they allow regions to scale up in isolation. Lastly, they offer low end-to-end queueing delay when producers and consumers are colocated with FOQS.</span></p>
<h3><span style="font-weight: 400;">Challenges of regional installations</span></h3>
<p><span style="font-weight: 400;">During any event leading to the unavailability of a primary replica in region X, MySQL can elect secondary replicas from region Y or Z as the new primary. After failover, the queue service in region X must send queries to the new primary database in region Y or Z where cross-region latencies increase up to hundreds of milliseconds.</span></p>
<p><span style="font-weight: 400;">In the event of complete network connectivity loss in region X, the limitations of the regional architecture become more glaring: </span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Customers have to explicitly balance their traffic away from an impacted region. Regional installations force all of the disaster recovery and load-balancing complexity to clients.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Pushing load-balancing complexity to customers often results in underutilization of system resources within FOQS. Additionally, it requires buffer capacity in other regions to accommodate shifted traffic.</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Due to an inability for MySQL primaries to be picked up outside of a given regional installation, items stored on replicas in the primary regions are susceptible to being “stuck” until connectivity is restored. Regional installations fail to offer access to data with high availability.</span></li>
</ul>
<figure id="attachment_18464" aria-describedby="caption-attachment-18464" style="width: 1080px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="size-full wp-image-18464" src="https://engineering.fb.com/wp-content/uploads/2022/01/Regional_installations.gif" alt="A visualization of regional installation’s inability to utilize capacity in the event of connectivity loss to region X. Notice how the MySQL shards are left isolated despite failing over to a secondary region." width="1080" height="608" /><figcaption id="caption-attachment-18464" class="wp-caption-text">A visualization of regional installation’s inability to utilize capacity in the event of connectivity loss to region X. Notice how the MySQL shards are left isolated despite failing over to a secondary region.</figcaption></figure>
<p><span style="font-weight: 400;">FOQS faced the gamut of challenges during a multiday disaster in late 2018 due to Hurricane Florence. The team was forced to employ manual steps aimed at rebalancing traffic, adding capacity to accommodate shifts and coordinating with customers to prevent substantial impact.</span></p>
<h2><span style="font-weight: 400;">Global installations</span></h2>
<p><span style="font-weight: 400;">Following Hurricane Florence, we quickly realized how unsustainable such actions would be as the teams and systems involved continued to scale. The team introduced global installations to better capitalize on FOQS’s MySQL replication topology and tolerate events that can affect a given region’s availability. Two major factors enabled our system to better tolerate disasters: following MySQL primaries on failovers and introducing an intelligent routing component as a service.</span></p>
<h3><span style="font-weight: 400;">Following the primary</span></h3>
<p><span style="font-weight: 400;">Global installations utilize global application primitives that exist in </span><a href="https://engineering.fb.com/2020/08/24/production-engineering/scaling-services-with-shard-manager/"><span style="font-weight: 400;">Shard Manager</span></a><span style="font-weight: 400;"> to support colocating MySQL primaries with a queue node in the corresponding region after failovers. For example, global installations enable FOQS to transition management of a shard that failed over from region X to region Y to a queue node in region Y. The failover and transition of management typically happen within seconds. </span></p>
<p><span style="font-weight: 400;">Since MySQL primaries are “followed” when they failover, global installations greatly increase the availability of data stored on MySQL shards and avoid two major issues observed with regional installations: stuck queue items and high cross-region latencies. Clients also benefit since no explicit action is required during a disaster, given that MySQL shards are always available via a queue node and accessible behind a single global service ID.</span></p>
<figure id="attachment_18466" aria-describedby="caption-attachment-18466" style="width: 1080px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="size-full wp-image-18466" src="https://engineering.fb.com/wp-content/uploads/2022/01/Global_installations.gif" alt="A high-level overview of a global installation. Notice the single, unified global tier, which is capable of utilizing MySQL shards in the event of connectivity loss to region X and failover of the shards to a secondary region." width="1080" height="608" /><figcaption id="caption-attachment-18466" class="wp-caption-text">A high-level overview of a global installation. Notice the single, unified global tier, which is capable of utilizing MySQL shards in the event of connectivity loss to region X and failover of the shards to a secondary region.</figcaption></figure>
<h3><span style="font-weight: 400;">Introduction of routing service</span></h3>
<p><span style="font-weight: 400;">For optimal latencies on global installations, FOQS added the routing service as a proxy between its clients and queue service. The routing service abstracts away all physical routing concerns from clients and enables optimal placement of items across a global queueing layer by exposing logical region preferences for routing.</span></p>
<h4><span style="font-weight: 400;">Enqueue distribution</span></h4>
<p><span style="font-weight: 400;">Enqueue distributor is a component in the routing service that handles optimal data placement with respect to regional preferences. The component maintains in-memory state to map logical region preferences to associated MySQL shards based on proximity. The enqueue API enables clients to specify logical regional preferences to hint where they’d like their items stored. </span></p>
<p><span style="font-weight: 400;">Upon receiving a request, the enqueue distributor will utilize its in-memory state to build a selection of queue nodes that are eligible to receive a request based on the specified region preference. Items utilizing region preferences are enqueued to their physical counterpart, if possible, or a region managing the associated failed-over MySQL shards. If no region preference is specified, items are typically routed within or near the region in which the request was received.</span></p>
<figure id="attachment_18467" aria-describedby="caption-attachment-18467" style="width: 1080px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="size-full wp-image-18467" src="https://engineering.fb.com/wp-content/uploads/2022/01/Enqueue_distributor.gif" alt="A visualization of an enqueue request with a region preference of X passing through the system when all nodes are available and no MySQL shards are failed over." width="1080" height="608" /><figcaption id="caption-attachment-18467" class="wp-caption-text">A visualization of an enqueue request with a region preference of X passing through the system when all nodes are available and no MySQL shards are failed over.</figcaption></figure>
<p><span style="font-weight: 400;">In disaster scenarios, global installations will continue accepting enqueue requests with a logical region preference matching an impacted region. Unlike regional installations, the loss of a region is handled transparently to clients. Instead of a request outright failing due to a physical region being unavailable, global installations route requests to wherever the corresponding MySQL capacity may be. </span></p>
<figure id="attachment_18465" aria-describedby="caption-attachment-18465" style="width: 1080px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="wp-image-18465 size-full" src="https://engineering.fb.com/wp-content/uploads/2022/01/RegionX_loss_enqueue.gif" alt="A visualization of an enqueue request with a region preference of X passing through the system when the physical region of X is unavailable. Notice how the item makes its way to a failed-over MySQL shard." width="1080" height="608" /><figcaption id="caption-attachment-18465" class="wp-caption-text">A visualization of an enqueue request with a region preference of X passing through the system when the physical region of X is unavailable. Notice how the item makes its way to a failed-over MySQL shard.</figcaption></figure>
<h4><span style="font-weight: 400;">Dequeue distribution</span></h4>
<p><span style="font-weight: 400;">Given that FOQS does not dictate a fixed placement of enqueued items associated with a logical queue, it requires a way to discover and dequeue “ready” queue items without significant delays. Dequeue distributor is the component that handles item discovery. </span></p>
<p><span style="font-weight: 400;">At a high level, each routing service node maintains an in-memory cache of queue nodes with items ready to be processed. The cache is represented as a heap that prioritizes all the queue nodes with items ready for consumption keyed by topic IDs. Dequeue distributor utilizes the cache in order to direct dequeue requests for a targeted topic ID to an appropriate queue host in the associated heap.</span></p>
<figure id="attachment_18468" aria-describedby="caption-attachment-18468" style="width: 1080px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="size-full wp-image-18468" src="https://engineering.fb.com/wp-content/uploads/2022/01/Dequeue_distributor.gif" alt="A visualization of the dequeue distributor gathering information about ready items across all queue nodes when all nodes are available and no MySQL shards are failed over. " width="1080" height="608" /><figcaption id="caption-attachment-18468" class="wp-caption-text">A visualization of the dequeue distributor gathering information about ready items across all queue nodes when all nodes are available and no MySQL shards are failed over.</figcaption></figure>
<p><span style="font-weight: 400;">Like the enqueue distributor, the dequeue distributor enables clients to continue dequeuing items associated with a topic ID and logical region preference even in the event of unavailability of an associated physical region. </span></p>
<figure id="attachment_18469" aria-describedby="caption-attachment-18469" style="width: 1080px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="size-full wp-image-18469" src="https://engineering.fb.com/wp-content/uploads/2022/01/RegionX_loss_dequeue.gif" alt="A visualization of the dequeue distributor gathering information about ready items across all queue nodes when the physical region X is unavailable. Notice that the failed-over MySQL shards are still available and being checked." width="1080" height="608" /><figcaption id="caption-attachment-18469" class="wp-caption-text">A visualization of the dequeue distributor gathering information about ready items across all queue nodes when the physical region X is unavailable. Notice that the failed-over MySQL shards are still available and being checked.</figcaption></figure>
<h2><span style="font-weight: 400;">Migration at scale</span></h2>
<p><span style="font-weight: 400;">Global installations have enabled the FOQS system to react to disasters in an elegant manner,  entirely eliminating complexity for its clients while enabling higher data availability.</span></p>
<p><span style="font-weight: 400;">With the shift to global installations, the FOQS team added the routing service into its architecture. Despite the many benefits of the routing service, it introduced significant challenges when migrating FOQS’s biggest use case, Async.</span></p>
<h3><span style="font-weight: 400;">Item discoverability</span></h3>
<p><span style="font-weight: 400;">As the queueing layer was scaled out to accommodate Async’s traffic, the dequeue distributor struggled to maintain a fresh cache and optimally spread dequeue requests to queue nodes. Longer queuing delays and latency SLO violations ensued for Async as a result of the suboptimal routing of dequeue requests.</span></p>
<p><span style="font-weight: 400;">To address stale cache challenges, we parallelized cache replenishment aggregation being performed by routing service. As a result, replenishment latency was nearly halved, and occurrences of routing requests to stale queue nodes were greatly reduced.</span></p>
<p><span style="font-weight: 400;">With cache replenishment reduced, we began to observe other routing related issues: thundering herd and starvation. By default, the dequeue distributor is reliant upon an algorithm that directs requests to the top-k queue nodes in its cache. Each routing service node maintains a similar cache of logical queue IDs, ordered by priority. Queue nodes at the top of the caches have the highest probability of receiving dequeue requests. As traffic increased, we began to see thundering herd problems for nodes at the top of the cache and starvation for those at the bottom. Considering that physical distance is factored into the cache’s ordering scheme, starvation issues were most pronounced when routing cross-region from routing service to queue service.</span></p>
<p><span style="font-weight: 400;">To remediate routing challenges, randomization among the already prioritized queue nodes was added on the dequeue path during host selection. Randomly selecting among the prioritized queue services nodes in the dequeue distributor’s internal cache resulted in a better spread of requests and thus reduced queueing delay.</span></p>
<h3><span style="font-weight: 400;">Massive migration with zero downtime</span></h3>
<p><span style="font-weight: 400;">Migrating Async’s largest use case to a global installation was not without challenges and risks. With zero downtime for our clients, the team managed to complete the migration in mid-2021 by adopting several key risk-mitigation strategies:  </span></p>
<ul>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Utilizing a shadow environment to load and stress test the new architecture</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Establishing and leveraging a simple config-based rollout mechanism to enable quickly rolling forward and backward with regular increases</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Demonstrating a strong bias for investigating issues as they arise and performing retrospectives to understand root-causes and avoid past mistakes</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Developing high-level dashboards with clear metrics denoting the migration’s health and status in order to keep stakeholders up-to-date</span></li>
</ul>
<p><span style="font-weight: 400;">For teams facing similar migrations, we hope that the practices applied to the migration from regional to global queue installations serve as inspiration to assist with your own challenges.</span></p>
<p><span style="font-weight: 400;">Migrating the scale that Async and FOQS handle proved to be a monumental challenge. With the migration of Async’s traffic to a global FOQS installation, we’ve observed much better resilience to disaster scenarios, and human involvement in response to events affecting entire regions is now essentially zero. In sum, </span><span style="font-weight: 400;">we’ve built a globally available distributed queue at Meta scale that enables access to data with high availability. The system is capable of recovering from the loss of an entire region within a matter of seconds, automatically and unbeknown to clients. </span></p>
<p><i><span style="font-weight: 400;">Substantial technical and operational challenges were faced on the journey to improve the resiliency of FOQS. We would like to extend thanks to all members of the teams that have supported FOQS’s success. A special thank-you to the core team members who heroically tamed the migration woes detailed here: Nate Ackerman, James Anderson, Dylan Cauwels, Bo Huang, Girish Joshi, Manukranth Kolloju, Kevin Liu, Pavani Panakanti, and CQ Tang.</span></i></p>
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-facebook"><a rel="nofollow noopener noreferrer"
data-shared="sharing-facebook-18459"
class="share-facebook sd-button share-icon"
href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/?share=facebook"
target="_blank"
aria-labelledby="sharing-facebook-18459"
>
<span id="sharing-facebook-18459" hidden>Share on Facebook (Opens in new window)</span>
<span>Facebook</span>
</a></li><li class="share-threads"><a rel="nofollow noopener noreferrer"
data-shared="sharing-threads-18459"
class="share-threads sd-button share-icon"
href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/?share=threads"
target="_blank"
aria-labelledby="sharing-threads-18459"
>
<span id="sharing-threads-18459" hidden>Share on Threads (Opens in new window)</span>
<span>Threads</span>
</a></li><li class="share-jetpack-whatsapp"><a rel="nofollow noopener noreferrer"
data-shared="sharing-whatsapp-18459"
class="share-jetpack-whatsapp sd-button share-icon"
href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/?share=jetpack-whatsapp"
target="_blank"
aria-labelledby="sharing-whatsapp-18459"
>
<span id="sharing-whatsapp-18459" hidden>Share on WhatsApp (Opens in new window)</span>
<span>WhatsApp</span>
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
data-shared="sharing-linkedin-18459"
class="share-linkedin sd-button share-icon"
href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/?share=linkedin"
target="_blank"
aria-labelledby="sharing-linkedin-18459"
>
<span id="sharing-linkedin-18459" hidden>Share on LinkedIn (Opens in new window)</span>
<span>LinkedIn</span>
</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
data-shared="sharing-reddit-18459"
class="share-reddit sd-button share-icon"
href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/?share=reddit"
target="_blank"
aria-labelledby="sharing-reddit-18459"
>
<span id="sharing-reddit-18459" hidden>Share on Reddit (Opens in new window)</span>
<span>Reddit</span>
</a></li><li class="share-x"><a rel="nofollow noopener noreferrer"
data-shared="sharing-x-18459"
class="share-x sd-button share-icon"
href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/?share=x"
target="_blank"
aria-labelledby="sharing-x-18459"
>
<span id="sharing-x-18459" hidden>Share on X (Opens in new window)</span>
<span>X</span>
</a></li><li class="share-bluesky"><a rel="nofollow noopener noreferrer"
data-shared="sharing-bluesky-18459"
class="share-bluesky sd-button share-icon"
href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/?share=bluesky"
target="_blank"
aria-labelledby="sharing-bluesky-18459"
>
<span id="sharing-bluesky-18459" hidden>Share on Bluesky (Opens in new window)</span>
<span>Bluesky</span>
</a></li><li class="share-mastodon"><a rel="nofollow noopener noreferrer"
data-shared="sharing-mastodon-18459"
class="share-mastodon sd-button share-icon"
href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/?share=mastodon"
target="_blank"
aria-labelledby="sharing-mastodon-18459"
>
<span id="sharing-mastodon-18459" hidden>Share on Mastodon (Opens in new window)</span>
<span>Mastodon</span>
</a></li><li class="share-custom share-custom-hackernews"><a rel="nofollow noopener noreferrer"
data-shared="sharing-custom-18459"
class="share-custom share-custom-hackernews sd-button share-icon"
href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/?share=custom-1699562127"
target="_blank"
aria-labelledby="sharing-custom-18459"
>
<span id="sharing-custom-18459" hidden>Share on Hacker News (Opens in new window)</span>
<span style="background-image:url(&quot;https://news.ycombinator.com/y18.svg&quot;);">Hacker News</span>
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
data-shared="sharing-email-18459"
class="share-email sd-button share-icon"
href="mailto:?subject=%5BShared%20Post%5D%20FOQS%3A%20Making%20a%20distributed%20priority%20queue%20disaster-ready&#038;body=https%3A%2F%2Fengineering.fb.com%2F2022%2F01%2F18%2Fproduction-engineering%2Ffoqs-disaster-ready%2F&#038;share=email"
target="_blank"
aria-labelledby="sharing-email-18459"
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you&#039;re having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="b808750730" data-email-share-track-url="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/?share=email">
<span id="sharing-email-18459" hidden>Email a link to a friend (Opens in new window)</span>
<span>Email</span>
</a></li><li class="share-end"></li></ul></div></div></div>
</div><!-- .entry-content -->
</article><!-- #post-## -->
<footer class="entry-footer">
</footer><!-- .entry-footer -->
<div class="fb-like"
data-href="https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/"
data-width=""
data-layout="standard"
data-action="like"
data-size="small"
data-share="true">
</div>
<div class="related-articles">
<div class="related-articles__header">
<h3 class="related-articles__header__headline">Read More in Production Engineering</h3>
<a
aria-label="View all articles in this Production Engineering"
class="related-articles__header__link"
href="https://engineering.fb.com/category/production-engineering/"
>
View All </a>
</div>
<div class="article-grids related-articles__grid">
<div class="row">
<div class="article-grid col-12 col-md-4">
<article class="post post-18459 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24394">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/Meta-MTIA-300-Hero-final.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Meta MTIA 300 - Hero - final" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-08-24T10:45:52-07:00">
AUG 24, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/mtia-300-meta-training-chip-built-in-nics/" rel="bookmark" aria-label="Read More about MTIA 300: Meta&#8217;s First Training Chip with Built-in NICs and Communication-Offloading Engines">
MTIA 300: Meta&#8217;s First Training Chip with Built-in NICs and Communication-Offloading Engines </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18459 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-23996">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/05/Meta-Mult-stage-Ads-Ranking.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-08-05T12:20:20-07:00">
AUG 5, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/08/05/ml-applications/from-user-sequences-to-scaling-laws-a-multi-stage-architecture-for-metas-ads-ranking/" rel="bookmark" aria-label="Read More about From User Sequences to Scaling Laws: A Multi-Stage Architecture for Meta’s Ads Ranking">
From User Sequences to Scaling Laws: A Multi-Stage Architecture for Meta’s Ads Ranking </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18459 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24150">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Metas-AI-Storage-Blueprint-at-Scale-Hero-1.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-07-01T09:00:36-07:00">
JUL 1, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/07/01/data-infrastructure/metas-ai-storage-blueprint-at-scale/" rel="bookmark" aria-label="Read More about Meta&#8217;s AI Storage Blueprint at Scale">
Meta&#8217;s AI Storage Blueprint at Scale </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18459 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24128">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/10-Years-Meta-Python-HERO-large.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-30T09:00:46-07:00">
JUN 30, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/30/open-source/10-years-of-metas-commitment-to-python/" rel="bookmark" aria-label="Read More about 10 Years of Meta’s Commitment to Python">
10 Years of Meta’s Commitment to Python </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18459 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24113">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Meta-Tech-Podcast-How-Meta-Built-Ultra-Narrow-Batteries-for-AI-Glasses-HERO.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-23T09:00:38-07:00">
JUN 23, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/23/production-engineering/how-meta-built-ultra-narrow-batteries-for-ai-glasses-meta-tech-podcast/" rel="bookmark" aria-label="Read More about How Meta Engineered Ultra-Narrow Batteries for AI Glasses">
How Meta Engineered Ultra-Narrow Batteries for AI Glasses </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18459 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24022">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Validating-PowerLoss-Readiness-Meta-hero.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-03T10:00:44-07:00">
JUN 3, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/03/data-center-engineering/lights-out-systems-on-validating-instant-power-loss-readiness/" rel="bookmark" aria-label="Read More about Lights Out, Systems On: Validating Instant Power Loss Readiness">
Lights Out, Systems On: Validating Instant Power Loss Readiness </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
</div><!-- /.row -->
</div><!--/.article-grids -->
</div><!-- /.related-articles -->
<!-- include the carousel nav here. -->
</main><!-- #main -->
</div><!-- #primary -->
<!-- Do the right sidebar check. -->
<div class="col-md-4 widget-area" id="right-sidebar" role="complementary">
<aside id="fb_related_post_widget-4" class="widget widget_fb_related_post_widget"><h3 class="widget-title">Related Posts</h3><hr>
<div class="aside-related-projects"><div class="article-grid-redesign">
<article class="post-18420 post type-post status-publish has-post-thumbnail hentry" id="post-18420">
<a href="https://engineering.fb.com/2021/12/16/data-center-engineering/power-loss-siren/" rel="bookmark" aria-label="Read More about Power Loss Siren: Making Meta resilient to power loss events">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2021/12/Eng-Blog-Self-Serve-Hero-Images-DATACENTERS-501-Teale.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2021/12/Eng-Blog-Self-Serve-Hero-Images-DATACENTERS-501-Teale.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2021/12/Eng-Blog-Self-Serve-Hero-Images-DATACENTERS-501-Teale.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Dec 16, 2021">Dec 16, 2021</time></span>
<h4 class="entry-title">Power Loss Siren: Making Meta resilient to power loss events</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div><div class="article-grid-redesign">
<article class="post-17308 post type-post status-publish has-post-thumbnail hentry" id="post-17308">
<a href="https://engineering.fb.com/2021/02/22/production-engineering/foqs-scaling-a-distributed-priority-queue/" rel="bookmark" aria-label="Read More about FOQS: Scaling a distributed priority queue">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2021/02/CD21_047_ENGBlog_FOQS_HERO_FINAL.png?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2021/02/CD21_047_ENGBlog_FOQS_HERO_FINAL.png?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2021/02/CD21_047_ENGBlog_FOQS_HERO_FINAL.png?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Feb 22, 2021">Feb 22, 2021</time></span>
<h4 class="entry-title">FOQS: Scaling a distributed priority queue</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div><div class="article-grid-redesign">
<article class="post-17580 post type-post status-publish has-post-thumbnail hentry" id="post-17580">
<a href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/" rel="bookmark" aria-label="Read More about How Facebook deals with PCIe faults to keep our data centers running reliably">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2021/05/Facebook010_data-center.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2021/05/Facebook010_data-center.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2021/05/Facebook010_data-center.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Jun 02, 2021">Jun 02, 2021</time></span>
<h4 class="entry-title">How Facebook deals with PCIe faults to keep our data centers running reliably</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div></div></aside><aside id="wpb_widget-3" class="widget widget_wpb_widget"><h3 class="widget-title">Related Positions</h3><hr><ul class="available-positions-list"> </ul>
<a href="https://www.metacareers.com">
<div class="btn available-positions-btn">
See All Jobs </div>
</a>
</aside>
</div><!-- #secondary -->
<!-- <php get_template_part('content-single-side-bar'); ?> -->
<!-- Do the right sidebar check. -->
</div><!-- Container end -->
</div><!-- Wrapper end -->
</div>
<!-- ******************* The Footer Full-width Widget Area ******************* -->
<div class="wrapper wrapper-community-outer">
<div class="wrapper-community">
<div class="community" tabindex="-1">
<div class="row">
<!-- community section is generated through WordPress widget -->
<div id="wpb_widget-2" class="footer-widget widget_wpb_widget widget-count-3 col-md-4"><h3 class="widget-title">Available Positions</h3><hr><ul class="available-positions-list"> </ul>
<a href="https://www.metacareers.com">
<div class="btn available-positions-btn">
See All Jobs </div>
</a>
</div><!-- .footer-widget --><div id="custom_html-2" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Technology at Meta</h3><div class="textwidget custom-html-widget"><ul class="stay-connected-lists">
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-fb-engineering" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Engineering at Meta - X
</div>
<a onclick='return twitterFollow("Meta_Engineers")'>
<button class="btn stay-connected-btn">
Follow
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-AI" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
AI at Meta
</div>
<a href="https://ai.meta.com/blog/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta Quest Blog
</div>
<a href="https://www.meta.com/blog/quest/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta for Developers
</div>
<a href="https://developers.facebook.com/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-bug-bounty" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta Bug Bounty
</div>
<a href="https://bugbounty.meta.com/">
<button class="btn stay-connected-btn">
Learn more
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/rss.png" alt="footer-rss" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
RSS
</div>
<a href="https://code.facebook.com/posts/rss/">
<button class="btn stay-connected-btn">
Subscribe
</button>
</a>
</div>
</li>
</ul>
</div></div><!-- .footer-widget --><div id="custom_html-3" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Open Source</h3><div class="textwidget custom-html-widget"><p class="opensource-desc">
Meta believes in building community through open source technology. Explore our latest projects in Artificial Intelligence, Data Infrastructure, Development Tools, Front End, Languages, Platforms, Security, Virtual Reality, and more.
</p>
<ul class="opensource-cats">
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/android.png" alt="android" loading="lazy" width="50" height="59">
<div class="opensource-cap">
ANDROID
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/ios.png" alt="ios" loading="lazy" width="48" height="58">
<div class="opensource-cap">
iOS
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/web.png" alt="web" loading="lazy" width="55" height="49">
<div class="opensource-cap">
WEB
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/backend.png" alt="backend" loading="lazy" width="56" height="51">
<div class="opensource-cap">
BACKEND
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/hardware.png" alt="hardware" loading="lazy" width="55" height="44">
<div class="opensource-cap">
HARDWARE
</div>
</li>
</ul>
<button class="btn opensource-btn" onclick="window.location.href='https://opensource.fb.com/'">
Learn More
</button>
</div></div><!-- .footer-widget -->
</div>
</div>
</div>
</div><!-- #wrapper-footer-full -->
<div class="wrapper-footer" id="wrapper-footer">
<div class="inner-container">
<div class="footer">
<div class="row">
<div class="col-md-12">
<footer class="meta-footer" id="meta-footer">
<div class="site-info row justify-content-between">
<div class="col-md-5 meta-logo-container">
<a href="https://about.facebook.com/">
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/meta_logo_full.svg" width="89" height="18" alt="Meta" />
</a>
<div id="text-2" class="widget widget_text"> <div class="textwidget"><p>Engineering at Meta is a technical news resource for engineers interested in how we solve large-scale technical challenges at Meta.</p>
</div>
</div> </div>
<div class="col-md-6 footer-meta-links-container">
<ul class="meta-footer-links">
<li class="footer-link"><a href="https://engineering.fb.com">Home</a></li><li class="footer-link"><a href="https://about.meta.com/">Company Info</a></li><li class="footer-link"><a href="https://www.metacareers.com/?ref=engineering.fb.com">Careers</a></li> </ul>
</div>
</div>
</footer>
</div>
</div>
</div>
</div>
<div class="footer">
<div class="row">
<div class="col-md-12">
<footer class="site-footer" id="colophon">
<div class="inner-container">
<div class="site-info row justify-content-between">
<div class="col-md-4 copyright-container">
<div class="copyright"> © 2026 Meta</div>
</div>
<!-- static footer -->
<div class="col-md-7 footer-links-container">
<ul class="footer-links">
<li class="footer-link"><a href="https://www.facebook.com/policies">Terms</a></li><li class="footer-link"><a href="https://www.facebook.com/privacy/policy">Privacy</a></li><li class="footer-link"><a href="/privacy">Cookies</a></li><li class="footer-link"><a href="https://www.facebook.com/help">Help</a></li> </ul>
</div>
</div>
</div><!-- .site-info -->
</footer><!-- #colophon -->
</div><!--col end -->
</div><!-- row end -->
</div><!-- //.inner-container -->
</div><!-- wrapper end -->
</div><!-- #page we need this extra closing tag here -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/code-fb-com/*","/wp-content/themes/understrap/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<div class="gdprconsent-container" id="GDPRConsentBar">
<div class="gdprconsent-wrapper">
<div class="gdprconsent-content">
<p>To help personalize content, tailor and measure ads and provide a safer experience, we use cookies. By clicking or navigating the site, you agree to allow our collection of information on and off Facebook through cookies. Learn more, including about available controls: <a href="/privacy">Cookie Policy</a></p>
<div class="gdprconsent-button">
<button type="button" data-gdpr-accept>Accept</button>
</div>
</div>
</div>
</div>
<!-- Twitter universal website tag code -->
<script>
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
// Insert Twitter Pixel ID and Standard Event data below
twq('init','o31u0');
twq('track','PageView');
});
}
</script>
<!-- End Twitter universal website tag code -->
<script type="text/javascript">
window.WPCOM_sharing_counts = {"https://engineering.fb.com/2022/01/18/production-engineering/foqs-disaster-ready/":18459};
</script>
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KZRLGZ8" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=660742068067366&ev=PageView&noscript=1"/></noscript>
<script id="child-understrap-scripts-js-extra">
var loadmore_params = {"restfulURL":"https://engineering.fb.com/wp-json/fb/v1/loadmore","searchText":"","posts":"{\"page\":0,\"year\":2022,\"monthnum\":1,\"day\":18,\"name\":\"foqs-disaster-ready\",\"category_name\":\"production-engineering\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","post_type":"post"};
var search_params = {"ajaxurl":"https://engineering.fb.com/wp-admin/admin-ajax.php","posts":"{\"page\":0,\"year\":2022,\"monthnum\":1,\"day\":18,\"name\":\"foqs-disaster-ready\",\"category_name\":\"production-engineering\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","article_filter":"2f23ad5ce3"};
var more_articles = {"next_article":{"label":"Next","permalink":"https://engineering.fb.com/2022/03/07/core-infra/augmenting-flexible-paxos-logdevice/","title":"Augmenting Flexible Paxos in LogDevice to improve read availability","thumbnail":["https://engineering.fb.com/wp-content/uploads/2022/02/LogDevice_Thumbnail.png?w=580&h=326&crop=1",580,326,true]},"prev_article":{"label":"Prev","permalink":"https://engineering.fb.com/2021/12/16/data-center-engineering/power-loss-siren/","title":"Power Loss Siren: Making Meta resilient to power loss events","thumbnail":["https://engineering.fb.com/wp-content/uploads/2021/12/Eng-Blog-Self-Serve-Hero-Images-DATACENTERS-501-Teale.jpg?w=580&h=326&crop=1",580,326,true]}};
//# sourceURL=child-understrap-scripts-js-extra
</script>
<script id="jetpack-facebook-embed-js-extra">
var jpfbembed = {"appid":"249643311490","locale":"en_US"};
//# sourceURL=jetpack-facebook-embed-js-extra
</script>
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJx9jUEOwiAQAD/kshJjqwfjU0xZtnZpF0iB+H2J8eTB62Qmg68MlGLlWLEurFyQkmeYXceKXkrF0Nkim4ePYFSiCeWA/8se5V2K/qraIG/tKbFg4JonWsEOxuJDIqFrfYPzROxSWoHVsf/+7nqz4+V6HM/DyYY3/BNA4A==" ></script><script id="sharing-js-js-extra">
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":""};
//# sourceURL=sharing-js-js-extra
</script>
<script id="sharing-js-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/_inc/build/sharedaddy/sharing.min.js?ver=16.1.3"></script>
<script id="sharing-js-js-after">
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-facebook' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-threads' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-threads' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomthreads', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-linkedin' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-x' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-x' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomx', 'menubar=1,resizable=1,width=600,height=350' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-bluesky' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-bluesky' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcombluesky', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-mastodon' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-mastodon' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcommastodon', 'menubar=1,resizable=1,width=460,height=400' );
return false;
}
} );
} )();
//# sourceURL=sharing-js-js-after
</script>
<script id="video-analytics-js-after">
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
let youtube = document.querySelectorAll('[src*="youtube-nocookie"]');
for (let i = 0; i < youtube.length; i++) {
youtube[i].src = youtube[i].src.replace('youtube-nocookie.com', 'youtube.com');
}
let vimeo = document.querySelectorAll('[src*="vimeo"]');
for (let i = 0; i < vimeo.length; i++) {
vimeo[i].src = vimeo[i].src.replace('&dnt=true', '');
}
});
}
//# sourceURL=video-analytics-js-after
</script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://engineering.fb.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://engineering.fb.com/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>