Files
nexus/sreweekly/articles/314/07-detecting-silent-errors-in-the-wild-combining-two-novel-approaches-to.html
2026-09-12 17:23:01 +08:00

1196 lines
98 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-title" content="Engineering at Meta - Engineering at Meta Blog">
<meta property="fb:app_id" content="1425766027653270" />
<link rel="profile" href="http://gmpg.org/xfn/11">
<link rel="pingback" href="https://engineering.fb.com/xmlrpc.php">
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
<script async src="https://engineering.fb.com/wp-content/themes/code-fb-com/dist/js/optimize.min.js"></script>
<!-- This site is optimized with the Yoast SEO Premium plugin v19.3 (Yoast SEO v27.3) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
<title>Detecting silent errors in the wild: Combining two novel approaches to quickly detect silent data corruptions at scale - Engineering at Meta</title>
<meta name="description" content="We provide methodologies to implement silent error detection testing at scale and compare results from two different novel approaches." />
<link rel="canonical" href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/" />
<meta name="twitter:label1" content="Written by" />
<meta name="twitter:data1" content="Harish Dattatraya Dixit" />
<meta name="twitter:label2" content="Est. reading time" />
<meta name="twitter:data2" content="8 minutes" />
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/#article","isPartOf":{"@id":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/"},"author":[{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/d9be7d06892b1301d9d6aab41ecca8a5"}],"headline":"Detecting silent errors in the wild: Combining two novel approaches to quickly detect silent data corruptions at scale","datePublished":"2022-03-17T16:00:49+00:00","dateModified":"2022-04-25T17:50:15+00:00","mainEntityOfPage":{"@id":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/"},"wordCount":1582,"publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"image":{"@id":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/03\/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-203-Orange.jpg","articleSection":["Production Engineering"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/","url":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/","name":"Detecting silent errors in the wild: Combining two novel approaches to quickly detect silent data corruptions at scale - Engineering at Meta","isPartOf":{"@id":"https:\/\/engineering.fb.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/#primaryimage"},"image":{"@id":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/03\/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-203-Orange.jpg","datePublished":"2022-03-17T16:00:49+00:00","dateModified":"2022-04-25T17:50:15+00:00","description":"We provide methodologies to implement silent error detection testing at scale and compare results from two different novel approaches.","breadcrumb":{"@id":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/#primaryimage","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/03\/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-203-Orange.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2022\/03\/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-203-Orange.jpg","width":1920,"height":1080},{"@type":"BreadcrumbList","@id":"https:\/\/engineering.fb.com\/2022\/03\/17\/production-engineering\/silent-errors\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/engineering.fb.com\/"},{"@type":"ListItem","position":2,"name":"Detecting silent errors in the wild: Combining two novel approaches to quickly detect silent data corruptions at scale"}]},{"@type":"WebSite","@id":"https:\/\/engineering.fb.com\/#website","url":"https:\/\/engineering.fb.com\/","name":"Engineering at Meta","description":"Engineering at Meta Blog","publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/engineering.fb.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/engineering.fb.com\/#organization","name":"Meta","url":"https:\/\/engineering.fb.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","width":29011,"height":12501,"caption":"Meta"},"image":{"@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/Engineering\/","https:\/\/x.com\/fb_engineering"]},[],{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/d9be7d06892b1301d9d6aab41ecca8a5","name":"Harish Dattatraya Dixit","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/d4f78ec81fa9cf745ea35e273a1c3a085ab1347fec2cc60beb9f4810d1779719?s=96&d=mm&r=gfdb9447c64fd426f9f26001feed6d21a","url":"https:\/\/secure.gravatar.com\/avatar\/d4f78ec81fa9cf745ea35e273a1c3a085ab1347fec2cc60beb9f4810d1779719?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/d4f78ec81fa9cf745ea35e273a1c3a085ab1347fec2cc60beb9f4810d1779719?s=96&d=mm&r=g","caption":"Harish Dattatraya Dixit"},"url":"https:\/\/engineering.fb.com\/author\/harish-dattatraya-dixit\/"}]}</script>
<!-- / Yoast SEO Premium plugin. -->
<link rel='dns-prefetch' href='//secure.gravatar.com' />
<link rel='dns-prefetch' href='//v0.wordpress.com' />
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta &raquo; Feed" href="https://engineering.fb.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta &raquo; Comments Feed" href="https://engineering.fb.com/comments/feed/" />
<script async src="https://www.googleoptimize.com/optimize.js?id=OPT-TPNDQC5" onerror="dataLayer.hide.end && dataLayer.hide.end()"></script>
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2022%2F03%2F17%2Fproduction-engineering%2Fsilent-errors%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2022%2F03%2F17%2Fproduction-engineering%2Fsilent-errors%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='all-css-12' href='https://engineering.fb.com/_static/??-eJyNjUsOwjAMRC9EaiqghQXiKFVqhzSQn+pEiNuTFJBAbNjZTzNv4BYFBp+UTxBt1sYzFKQpzpVz5WQ4QSUv0CDzCj6KaVJOMWAgJc5jwe7ZKTnAyVgSS6Jxxv90XRbv3YtKUeJVtF3TgguUbZHyJGdFkui+nMbrvxWD8QhjLvvAAY20wgYd+OupspM7tv3+sO533Wb7AImQY4U=' type='text/css' media='all' />
<script id="gdprconsent-js" src="https://engineering.fb.com/wp-content/plugins/wp-gdpr-consent/dist/gdprconsent.js?ver=2.0.0"></script>
<script id="gdprconsent-js-after">
if (window.gdprSafeTrack) {
window.gdprSafeTrack('https://www.googletagmanager.com/gtag/js?id=391063774');
window.gdprSafeTrack(function() {
var google_analytics_g4 = decodeURIComponent( '391063774' );
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', google_analytics_g4);
});
}
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
var google_tag_manager = decodeURIComponent( 'GTM-KZRLGZ8' );
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer',google_tag_manager);
});
}
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
var facebook_pixel = decodeURIComponent( '660742068067366' );
!function(f,b,e,v,n,t,s)
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
n.queue=[];t=b.createElement(e);t.async=!0;
t.src=v;s=b.getElementsByTagName(e)[0];
s.parentNode.insertBefore(t,s)}(window, document,'script',
'https://connect.facebook.net/en_US/fbevents.js');
fbq('init', facebook_pixel);
fbq('track', 'PageView');
});
}
//# sourceURL=gdprconsent-js-after
</script>
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJzTLy/QzcxLzilNSS3WzwKiwtLUokoopZebmaeXVayjj0+Rbm5melFiSSpUsX2uraG5hbmZgYmloUkWAK+3Iic=" ></script><script type="text/javascript" src="https://engineering.fb.com/wp-content/themes/understrap/js/popper.min.js?m=1773259670g" ></script><link rel="https://api.w.org/" href="https://engineering.fb.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.fb.com/wp-json/wp/v2/posts/18566" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.fb.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress 7.1" />
<link rel='shortlink' href='https://wp.me/pa0Lhq-4Ps' />
<link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/Optimistic_Display_W_Md.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="Detecting silent errors in the wild: Combining two novel approaches to quickly detect silent data corruptions at scale" />
<meta property="og:url" content="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/" />
<meta property="og:description" content="Silent data corruptions (SDCs), data errors that go undetected by the larger system, are a widespread problem for large-scale infrastructure systems. Left undetected, these types of corruptions can…" />
<meta property="article:published_time" content="2022-03-17T16:00:49+00:00" />
<meta property="article:modified_time" content="2022-04-25T17:50:15+00:00" />
<meta property="og:site_name" content="Engineering at Meta" />
<meta property="og:image" content="https://engineering.fb.com/wp-content/uploads/2022/03/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-203-Orange.jpg" />
<meta property="og:image:width" content="1920" />
<meta property="og:image:height" content="1080" />
<meta property="og:image:alt" content="" />
<meta property="og:locale" content="en_US" />
<meta name="twitter:text:title" content="Detecting silent errors in the wild: Combining two novel approaches to quickly detect silent data corruptions at scale" />
<meta name="twitter:image" content="https://engineering.fb.com/wp-content/uploads/2022/03/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-203-Orange.jpg?w=640" />
<meta name="twitter:card" content="summary_large_image" />
<!-- End Jetpack Open Graph Tags -->
<style id="wp-custom-css">
.table-of-contents {
position: sticky;
top: 20px;
background: #fff;
padding: 1em;
border: 1px solid #ddd;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
max-width: 250px;
}
</style>
<link rel="shortcut icon" href="https://engineering.fb.com/wp-content/themes/code-fb-com/favicon.ico" />
</head>
<body class="wp-singular post-template-default single single-post postid-18566 single-format-standard wp-theme-understrap wp-child-theme-code-fb-com group-blog">
<div class="hfeed site" id="page">
<!-- ******************* The Navbar Area ******************* -->
<div class="wrapper-fluid wrapper-navbar" id="wrapper-navbar">
<a class="skip-link screen-reader-text sr-only" href="#content">
Skip to content </a>
<nav class="navbar navbar-expand-std">
<div class="nav-container">
<!-- Your site title as branding in the menu -->
<div class="nav-top-row">
<h1 class="navbar-brand mb-0">
<a rel="home" href="https://engineering.fb.com/" title="Engineering at Meta">
<img class="navbar-brand-logo" src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/logo-meta.svg" class="navbar-brand-logo" alt="Engineering at Meta" width="215" height="40" loading="lazy">
</a>
</h1>
<!-- end custom logo -->
<label for="search" class="screen-reader-text">Search this site</label>
<input type="text" id="search" name="search" class="form-control-lg navbar-search" autocomplete="off">
<button class="navbar-toggler navbar-searchbar-toggler"
type="button"
data-toggle="collapse"
data-target="#searchbarDropdown"
aria-controls="searchbarDropdown"
aria-expanded="false"
aria-label="Toggle navigation">
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/icon-search.svg" class="navbar-mobile-search-icon" loading="lazy" width="24" height="25" />
</button>
<!-- desktop language menu (mobile language menu is implemented via jQuery)-->
<div class="notranslate nice-select navbar-language form-control">
</div>
<button class="navbar-toggler collapsed navbar-menu-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<div class="navbar-border"></div>
<!-- The WordPress Menu goes here -->
<div id="navbarNavDropdown" class="collapse navbar-collapse"><ul id="main-menu" class="navbar-nav"><li id="menu-item-60" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-60 dropdown"><a title="Open Source" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Open Source <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-12330" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-12330"><a title="Open Source" href="https://engineering.fb.com/category/open-source/" class="nav-link">Open Source</a></li>
<li id="menu-item-12331" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-12331"><a title="Meta Open Source" href="https://opensource.fb.com" class="nav-link">Meta Open Source</a></li>
</ul>
</li>
<li id="menu-item-2686" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2686 dropdown"><a title="Platforms" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Platforms <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-65" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-65"><a title="Android" href="https://engineering.fb.com/category/android/" class="nav-link">Android</a></li>
<li id="menu-item-66" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-66"><a title="iOS" href="https://engineering.fb.com/category/ios/" class="nav-link">iOS</a></li>
<li id="menu-item-67" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-67"><a title="Web" href="https://engineering.fb.com/category/web/" class="nav-link">Web</a></li>
</ul>
</li>
<li id="menu-item-2682" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2682 dropdown"><a title="Infrastructure Systems" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Infrastructure Systems <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-218" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-218"><a title="Core Infra" href="https://engineering.fb.com/category/core-infra/" class="nav-link">Core Infra</a></li>
<li id="menu-item-219" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-219"><a title="Data Infrastructure" href="https://engineering.fb.com/category/data-infrastructure/" class="nav-link">Data Infrastructure</a></li>
<li id="menu-item-220" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-220"><a title="DevInfra" href="https://engineering.fb.com/category/developer-tools/" class="nav-link">DevInfra</a></li>
<li id="menu-item-226" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-226"><a title="Production Engineering" href="https://engineering.fb.com/category/production-engineering/" class="nav-link">Production Engineering</a></li>
<li id="menu-item-227" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-227"><a title="Security &amp; Privacy" href="https://engineering.fb.com/category/security/" class="nav-link">Security &amp; Privacy</a></li>
<li id="menu-item-18607" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18607"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/systems-infrastructure/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2683" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2683 dropdown"><a title="Physical Infrastructure" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Physical Infrastructure <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-215" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-215"><a title="Connectivity" href="https://engineering.fb.com/category/connectivity/" class="nav-link">Connectivity</a></li>
<li id="menu-item-216" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-216"><a title="Data Center Engineering" href="https://engineering.fb.com/category/data-center-engineering/" class="nav-link">Data Center Engineering</a></li>
<li id="menu-item-217" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-217"><a title="Networking &amp; Traffic" href="https://engineering.fb.com/category/networking-traffic/" class="nav-link">Networking &amp; Traffic</a></li>
<li id="menu-item-18608" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18608"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/networking-connectivity/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2684" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2684 dropdown"><a title="Video Engineering &amp; AR/VR" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Video Engineering &amp; AR/VR <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-225" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-225"><a title="Video Engineering" href="https://engineering.fb.com/category/video-engineering/" class="nav-link">Video Engineering</a></li>
<li id="menu-item-223" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-223"><a title="Virtual Reality" href="https://engineering.fb.com/category/virtual-reality/" class="nav-link">Virtual Reality</a></li>
<li id="menu-item-13388" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13388"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/augmented-reality-virtual-reality/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2685" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2685 dropdown"><a title="Artificial Intelligence" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Artificial Intelligence <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-222" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-222"><a title="ML Applications" href="https://engineering.fb.com/category/ml-applications/" class="nav-link">ML Applications</a></li>
<li id="menu-item-221" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-221"><a title="AI Research" href="https://engineering.fb.com/category/ai-research/" class="nav-link">AI Research</a></li>
<li id="menu-item-13387" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13387"><a title="Research Publications" href="https://ai.facebook.com/results/?content_types%5B0%5D=publication" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-228" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-228"><a title="Watch Videos" href="/videos" class="nav-link">Watch Videos</a></li>
</ul></div> </div><!-- .container -->
<div class="search-container collapse" id="searchbarDropdown">
<div class="searchbar-search">
<div class="searchbar-container">
<div class="navbar-mobile-search-margin"></div>
<input type="text" name="search" placeholder="Search" class="form-control-lg navbar-mobile-search" autofocus autocomplete="off">
<button class="searchbar-clear">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
</div>
</div>
</nav><!-- .site-navigation -->
</div><!-- .wrapper-navbar end -->
<div class="wrapper" id="single-wrapper">
<div class="single-container row" id="content" tabindex="-1">
<!-- Do the left sidebar check. -->
<div class="content-area" id="primary">
<main class="site-main" id="main">
<article class="post-18566 post type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-18566">
<header class="entry-header">
<div class="entry-meta">
POSTED ON <span class="posted-on entry-date"><time class="published updated" datetime="2022-03-17">MARCH 17, 2022</time></span> TO <span class="cat-links"><a href="https://engineering.fb.com/category/production-engineering/" rel="category tag">Production Engineering</a></span>
<h1 class="entry-title"><div>Detecting silent errors in the wild: Combining two novel approaches to quickly detect silent data corruptions at scale</div></h1> </div><!-- .entry-meta -->
</header><!-- .entry-header -->
<figure id="post-feat-image-container">
<img width="1920" height="1080" src="https://engineering.fb.com/wp-content/uploads/2022/03/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-203-Orange.jpg" class="wp-post-image" alt="" title="Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-203-Orange" /> </figure>
<div class="entry-authors-social">
<div class="entry-authors">
By <a href="https://engineering.fb.com/author/harish-dattatraya-dixit/" title="Posts by Harish Dattatraya Dixit" class="author url fn" rel="author">Harish Dattatraya Dixit</a> </div>
</div>
<div class="entry-content">
<p><a href="https://engineering.fb.com/2021/02/23/data-infrastructure/silent-data-corruption/"><span style="font-weight: 400;">Silent data corruptions</span></a><span style="font-weight: 400;"> (SDCs), data errors that go undetected by the larger system, are a</span><a href="https://indico.cern.ch/event/13797/contributions/1362288/attachments/115080/163419/Data_integrity_v3.pdf"><span style="font-weight: 400;"> widespread problem</span></a><span style="font-weight: 400;"> for large-scale infrastructure systems. Left undetected, these types of corruptions can cause data loss and propagate across the stack and manifest as application-level problems.</span></p>
<p><span style="font-weight: 400;">Silent data corruptions (SDC) in hardware impact computational integrity for large-scale applications. Sources of corruptions include datapath dependencies, temperature variance, and age, among other silicon factors. These errors do not leave any record or trace in system logs. As a result, silent errors stay undetected within workloads and can propagate across several services. In this paper, we describe testing strategies to detect and mitigate silent data corruptions within a large-scale infrastructure. Given the challenging nature of the problem, we experimented with different methods for detection and mitigation. We compare and contrast two such approaches: 1) Fleetscanner (out-of-production testing) and 2) ripple (</span><span style="font-weight: 400;">in-production testing). We evaluate the infrastructure trade-offs associated with the silicon testing funnel across 3+ years of production experience.</span></p>
<h2><span style="font-weight: 400;">How silent error detection works</span></h2>
<p><span style="font-weight: 400;">Within a typical manufacturing cycle, a server or a CPU gets tested at the vendor for a few hours, then at the integrator for a couple of days (in the best case), as well as on a sampling basis. However, once the components go through to a production fleet, it becomes really challenging to implement testing at scale. As a result, we need novel detection approaches for preserving application health and fleet resiliency by detecting SDCs and mitigating them at scale. </span></p>
<p><span style="font-weight: 400;">Every iteration of a test takes time away from production workloads. Out-of-production testing has a ramp-up and ramp-down between switching from test to production workloads. In-production testing can leave residual configs that might adversely affect workload performance. When we scale this to an entire fleet, the variations as well as the time needed for testing become significant. With in-production testing, the challenge is smart colocation with minimal effects. With out-of-production testing, the challenge is utilizing downtimes from production effectively without affecting other maintenance tasks.</span></p>
<p><img decoding="async" class="alignnone size-large wp-image-18577" src="https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image4.jpg?w=1024" alt="" width="1024" height="576" srcset="https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image4.jpg 1999w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image4.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image4.jpg?resize=916,516 916w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image4.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image4.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image4.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image4.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image4.jpg?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">At Meta, we implement multiple methods to detect SDCs, the two most effective of which are opportunistic and periodic testing and in-production/ripple testing. Opportunistic testing has been live within the fleet for around 3 years. However, in evaluating the trade-offs from one to the other, we’ve determined that both approaches are equally important to detecting SDCs, and we recommend using and deploying both in a large-scale fleet. We’ve observed that in-production testing can, within 15 days, detect 70 percent of the fleet data corruptions that opportunistic testing may take about 6 months to detect. However, the last 30 percent of the faults are only detectable with opportunistic testing, thus making both methods equally important for fault detection.</span></p>
<p><a href="https://arxiv.org/pdf/2102.11245.pdf"><span style="font-weight: 400;">We’ve seen in our research</span></a><span style="font-weight: 400;"> that silent errors can happen during any set of functions within a data center CPU, and machines can encounter SDCs at different stages within the life cycle. Since these corruptions can cause latent problems to our infrastructure, we run a combination of proprietary and vendor-specific tests within our fleet to detect bad hardware.</span></p>
<p><span style="font-weight: 400;">We pursue two key methods to infrastructure testing:</span></p>
<ol>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Out-of-production testing (opportunistic testing)</span></li>
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">In-production testing (ripple testing)</span></li>
</ol>
<h3><span style="font-weight: 400;">Opportunistic testing </span></h3>
<p><img decoding="async" class="alignnone size-large wp-image-18587" src="https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image3_Update.jpg?w=1024" alt="" width="1024" height="576" srcset="https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image3_Update.jpg 1999w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image3_Update.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image3_Update.jpg?resize=916,516 916w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image3_Update.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image3_Update.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image3_Update.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image3_Update.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image3_Update.jpg?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /></p>
<p>&nbsp;</p>
<p><span style="font-weight: 400;">In a large-scale infrastructure, machines typically go through multiple maintenance events — either planned or unplanned maintenance. We rely on these maintenance events so we can piggyback on them and run SDC tests. System reboots, kernel upgrades, firmware upgrades, device reimages, host provisioning, and namespace reprovisioning and repairs are a few examples of maintenance events that have opportunistic testing enabled. We implement this mechanism using Fleetscanner, an internal facilitator tool for testing SDCs opportunistically. </span></p>
<p><span style="font-weight: 400;">Opportunistic testing allows for tests to have longer runtimes (on the order of minutes), enabling a more intrusive nature of detection. However, the inherent architecture of the opportunistic testing means it is possible to have corruptions in between two opportunistic test intervals on a suspect machine or CPU. We measured the testing cadence to evaluate the effectiveness of this testing mode. We’ve currently observed that through all the mechanisms described above, a machine goes through opportunistic testing an average of once every 180 days. This number is a combination of fleet population and upgrade cadences and varies based on machine type. There are fine-grained controls to enable more frequent opportunistic testing cadences on tiers and services that are sensitive.</span></p>
<p><span style="font-weight: 400;">Adding opportunistic testing to our fleet has meant that the majority of our maintenance activities have lengthened by the time it takes to conduct a test. As a result, we closely evaluate test time and testing frequencies for their detection efficacy. </span></p>
<p><span style="font-weight: 400;">Through this mechanism, we have achieved more than 68 million tests within the fleet, with a total test time machines have spent to be approximately 4 billion seconds opportunistically. </span></p>
<h2><span style="font-weight: 400;">Ripple testing</span></h2>
<p><img loading="lazy" decoding="async" class="alignnone size-large wp-image-18575" src="https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image2.jpg?w=1024" alt="" width="1024" height="576" srcset="https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image2.jpg 1999w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image2.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image2.jpg?resize=916,516 916w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image2.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image2.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image2.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image2.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2022/03/Detecting-Silent-Errors_image2.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">While opportunistic testing enabled frequent fleet coverage, it wasn’t sufficient to prevent production failures that manifested between testing windows, that were dependent on a different data pattern, or that happened because of the switch between different operating modes. </span></p>
<p><span style="font-weight: 400;">Ripple testing solves all the above problems by running silent error detection in conjunction with the workload. By carefully monitoring the nature of the workloads — shadow testing with the workloads and injecting bit patterns with expected results every now and then within the fleet (based on a predetermined cadence for a service) along with workloads — we can obtain coverage fleetwide at a significantly faster cadence. We enable granular controls to determine test intervals, test duration, and device configuration for tests to execute in colocation with the workloads. </span></p>
<p><span style="font-weight: 400;">Because the testing here “ripples” through our infrastructure, the test times are 1,000x lower than the opportunistic test runtimes. Ripple tests are typically in the order of hundreds of milliseconds within the fleet. They are scheduled based on workload behavior and can be switched on and off per workload. Because of the nature of defects and the scale-based approach of ripple testing, with unique seeds and creating more transition events, we are able to detect remote failures that could manifest only with thousands of iterations of the test. There is a footprint tax that we manage within the fleet to enable this testing, and we’ve observed it to be negligible in comparison with more frequent management activities. We shared our early results with CPU vendors, who have learned lessons from our fleet and implemented mechanisms within their tools to achieve similar results.</span></p>
<p><span style="font-weight: 400;">Through this mechanism, we run approximately 2.5 billion unique tests/test seeds within the fleet any given month. This framework has been live for the past couple of years within the fleet and has achieved a total test time of nearly 100 million seconds in colocation with the workloads. </span></p>
<h2><span style="font-weight: 400;">The benefits of combining opportunist and ripple testing</span></h2>
<p><span style="font-weight: 400;">While detecting SDCs is a challenging problem for large-scale infrastructures, years of testing have shown us that opportunistic and ripple testing can provide a novel solution for detecting SDCs at scale as quickly as possible.</span></p>
<table style="height: 220px;" border="1" width="780">
<tbody>
<tr>
<td><b>Metric</b></td>
<td><b>Opportunistic testing</b></td>
<td><b>Ripple testing </b></td>
</tr>
<tr>
<td><b>Seeds </b></td>
<td><span style="font-weight: 400;">~68 million (lifetime)</span></td>
<td><span style="font-weight: 400;">~2.5 billion (per month)</span></td>
</tr>
<tr>
<td><b>Testing time</b></td>
<td><span style="font-weight: 400;">~4 billion fleet </span><span style="font-weight: 400;">seconds (lifetime)</span></td>
<td><span style="font-weight: 400;">~100 million fleet </span><span style="font-weight: 400;">seconds (per month)</span></td>
</tr>
<tr>
<td><b>Performance aware</b></td>
<td><span style="font-weight: 400;">No</span></td>
<td><span style="font-weight: 400;">Yes</span></td>
</tr>
<tr>
<td><b>Unique SDC Coverage </b></td>
<td><span style="font-weight: 400;">23 percent</span></td>
<td><span style="font-weight: 400;">7 percent</span></td>
</tr>
<tr>
<td><b>Time to equivalent SDC coverage</b></td>
<td><span style="font-weight: 400;">~6 months (70 percent)</span></td>
<td><span style="font-weight: 400;">~15 days (70 percent)</span></td>
</tr>
</tbody>
</table>
<p>&nbsp;</p>
<p><span style="font-weight: 400;">A comparison of the numbers presented in the above two sections is provided in the table. We observe that for defect types categorized within this paper, around 70 percent of the common coverage detection could be completed within 15 days. The opportunistic testing could catch up to the remaining 23 percent of the unique faulty CPUs for 6 months, and the remaining 7 percent is through repeated ripple instances within the fleet. There are benefits to both models of testing. We also consistently revisit and evaluate these coverage metrics to inform and update our fleetwide testing strategies around test vectors, test cadences, and test runtimes. We anticipate that with different types of defects, the coverage split may vary.</span></p>
<p><span style="font-weight: 400;">Historically, each CPU went through only a few hours of testing as part of infrastructure burn-in tests. Further testing was typically conducted via sampling. We observe that novel detection approaches are required for application health and fleet resiliency. We demonstrate the ability to test at scale and get through billions of seconds of testing every month across a large fleet consistently. These novel techniques enable us to detect silent data corruptions and mitigate them at scale.</span></p>
<p><span style="font-weight: 400;">Detecting silent data corruption is a challenging problem for large-scale infrastructures. Applications show significant sensitivity to these problems and can be exposed to such corruptions for months without accelerated detection mechanisms. It can also result in data loss and require months to debug and resolve software-level residue of silent corruptions. This research shows novel techniques resulting from years of experience observing silent corruptions and in categorizing their occurrence patterns and faster time to detection. Impact of silent data corruption can have a cascading effect on applications, and we have to address this as a critical problem. As a result, detecting these at scale as quickly as possible is an infrastructure priority toward enabling a safer, reliable fleet.</span></p>
<h2><span style="font-weight: 400;">Request for proposals</span></h2>
<p><span style="font-weight: 400;">To foster further innovation in this area and to deepen our collaboration with academia, Meta is inviting faculty to respond to this </span><a href="https://research.facebook.com/research-awards/2022-silent-data-corruptions-at-scale-request-for-proposals/"><span style="font-weight: 400;">call for research proposals</span></a><span style="font-weight: 400;"> pertaining to the aforementioned topics. We anticipate giving up to five awards, each in the $50,000 range. Payment will be made to the proposer’s host university as an unrestricted gift. </span></p>
<p><a href="https://research.facebook.com/research-awards/2022-silent-data-corruptions-at-scale-request-for-proposals/"><span style="font-weight: 400;">Applications are open until March 22</span></a><span style="font-weight: 400;">. </span></p>
<h2><span style="font-weight: 400;">Read the full paper</span></h2>
<p class="title mathjax"><a href="https://arxiv.org/abs/2203.08989" target="_blank" rel="noopener">Detecting silent data corruptions in the wild</a></p>
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-facebook"><a rel="nofollow noopener noreferrer"
data-shared="sharing-facebook-18566"
class="share-facebook sd-button share-icon"
href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/?share=facebook"
target="_blank"
aria-labelledby="sharing-facebook-18566"
>
<span id="sharing-facebook-18566" hidden>Share on Facebook (Opens in new window)</span>
<span>Facebook</span>
</a></li><li class="share-threads"><a rel="nofollow noopener noreferrer"
data-shared="sharing-threads-18566"
class="share-threads sd-button share-icon"
href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/?share=threads"
target="_blank"
aria-labelledby="sharing-threads-18566"
>
<span id="sharing-threads-18566" hidden>Share on Threads (Opens in new window)</span>
<span>Threads</span>
</a></li><li class="share-jetpack-whatsapp"><a rel="nofollow noopener noreferrer"
data-shared="sharing-whatsapp-18566"
class="share-jetpack-whatsapp sd-button share-icon"
href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/?share=jetpack-whatsapp"
target="_blank"
aria-labelledby="sharing-whatsapp-18566"
>
<span id="sharing-whatsapp-18566" hidden>Share on WhatsApp (Opens in new window)</span>
<span>WhatsApp</span>
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
data-shared="sharing-linkedin-18566"
class="share-linkedin sd-button share-icon"
href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/?share=linkedin"
target="_blank"
aria-labelledby="sharing-linkedin-18566"
>
<span id="sharing-linkedin-18566" hidden>Share on LinkedIn (Opens in new window)</span>
<span>LinkedIn</span>
</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
data-shared="sharing-reddit-18566"
class="share-reddit sd-button share-icon"
href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/?share=reddit"
target="_blank"
aria-labelledby="sharing-reddit-18566"
>
<span id="sharing-reddit-18566" hidden>Share on Reddit (Opens in new window)</span>
<span>Reddit</span>
</a></li><li class="share-x"><a rel="nofollow noopener noreferrer"
data-shared="sharing-x-18566"
class="share-x sd-button share-icon"
href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/?share=x"
target="_blank"
aria-labelledby="sharing-x-18566"
>
<span id="sharing-x-18566" hidden>Share on X (Opens in new window)</span>
<span>X</span>
</a></li><li class="share-bluesky"><a rel="nofollow noopener noreferrer"
data-shared="sharing-bluesky-18566"
class="share-bluesky sd-button share-icon"
href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/?share=bluesky"
target="_blank"
aria-labelledby="sharing-bluesky-18566"
>
<span id="sharing-bluesky-18566" hidden>Share on Bluesky (Opens in new window)</span>
<span>Bluesky</span>
</a></li><li class="share-mastodon"><a rel="nofollow noopener noreferrer"
data-shared="sharing-mastodon-18566"
class="share-mastodon sd-button share-icon"
href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/?share=mastodon"
target="_blank"
aria-labelledby="sharing-mastodon-18566"
>
<span id="sharing-mastodon-18566" hidden>Share on Mastodon (Opens in new window)</span>
<span>Mastodon</span>
</a></li><li class="share-custom share-custom-hackernews"><a rel="nofollow noopener noreferrer"
data-shared="sharing-custom-18566"
class="share-custom share-custom-hackernews sd-button share-icon"
href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/?share=custom-1699562127"
target="_blank"
aria-labelledby="sharing-custom-18566"
>
<span id="sharing-custom-18566" hidden>Share on Hacker News (Opens in new window)</span>
<span style="background-image:url(&quot;https://news.ycombinator.com/y18.svg&quot;);">Hacker News</span>
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
data-shared="sharing-email-18566"
class="share-email sd-button share-icon"
href="mailto:?subject=%5BShared%20Post%5D%20Detecting%20silent%20errors%20in%20the%20wild%3A%20Combining%20two%20novel%20approaches%20to%20quickly%20detect%20silent%20data%20corruptions%20at%20scale&#038;body=https%3A%2F%2Fengineering.fb.com%2F2022%2F03%2F17%2Fproduction-engineering%2Fsilent-errors%2F&#038;share=email"
target="_blank"
aria-labelledby="sharing-email-18566"
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you&#039;re having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="806cd29ace" data-email-share-track-url="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/?share=email">
<span id="sharing-email-18566" hidden>Email a link to a friend (Opens in new window)</span>
<span>Email</span>
</a></li><li class="share-end"></li></ul></div></div></div>
</div><!-- .entry-content -->
</article><!-- #post-## -->
<footer class="entry-footer">
</footer><!-- .entry-footer -->
<div class="fb-like"
data-href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/"
data-width=""
data-layout="standard"
data-action="like"
data-size="small"
data-share="true">
</div>
<div class="related-articles">
<div class="related-articles__header">
<h3 class="related-articles__header__headline">Read More in Production Engineering</h3>
<a
aria-label="View all articles in this Production Engineering"
class="related-articles__header__link"
href="https://engineering.fb.com/category/production-engineering/"
>
View All </a>
</div>
<div class="article-grids related-articles__grid">
<div class="row">
<div class="article-grid col-12 col-md-4">
<article class="post post-18566 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24394">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/Meta-MTIA-300-Hero-final.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Meta MTIA 300 - Hero - final" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-08-24T10:45:52-07:00">
AUG 24, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/mtia-300-meta-training-chip-built-in-nics/" rel="bookmark" aria-label="Read More about MTIA 300: Meta&#8217;s First Training Chip with Built-in NICs and Communication-Offloading Engines">
MTIA 300: Meta&#8217;s First Training Chip with Built-in NICs and Communication-Offloading Engines </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18566 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-23996">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/05/Meta-Mult-stage-Ads-Ranking.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-08-05T12:20:20-07:00">
AUG 5, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/08/05/ml-applications/from-user-sequences-to-scaling-laws-a-multi-stage-architecture-for-metas-ads-ranking/" rel="bookmark" aria-label="Read More about From User Sequences to Scaling Laws: A Multi-Stage Architecture for Meta’s Ads Ranking">
From User Sequences to Scaling Laws: A Multi-Stage Architecture for Meta’s Ads Ranking </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18566 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24150">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Metas-AI-Storage-Blueprint-at-Scale-Hero-1.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-07-01T09:00:36-07:00">
JUL 1, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/07/01/data-infrastructure/metas-ai-storage-blueprint-at-scale/" rel="bookmark" aria-label="Read More about Meta&#8217;s AI Storage Blueprint at Scale">
Meta&#8217;s AI Storage Blueprint at Scale </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18566 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24128">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/10-Years-Meta-Python-HERO-large.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-30T09:00:46-07:00">
JUN 30, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/30/open-source/10-years-of-metas-commitment-to-python/" rel="bookmark" aria-label="Read More about 10 Years of Meta’s Commitment to Python">
10 Years of Meta’s Commitment to Python </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18566 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24113">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Meta-Tech-Podcast-How-Meta-Built-Ultra-Narrow-Batteries-for-AI-Glasses-HERO.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-23T09:00:38-07:00">
JUN 23, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/23/production-engineering/how-meta-built-ultra-narrow-batteries-for-ai-glasses-meta-tech-podcast/" rel="bookmark" aria-label="Read More about How Meta Engineered Ultra-Narrow Batteries for AI Glasses">
How Meta Engineered Ultra-Narrow Batteries for AI Glasses </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-18566 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24022">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Validating-PowerLoss-Readiness-Meta-hero.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-03T10:00:44-07:00">
JUN 3, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/03/data-center-engineering/lights-out-systems-on-validating-instant-power-loss-readiness/" rel="bookmark" aria-label="Read More about Lights Out, Systems On: Validating Instant Power Loss Readiness">
Lights Out, Systems On: Validating Instant Power Loss Readiness </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
</div><!-- /.row -->
</div><!--/.article-grids -->
</div><!-- /.related-articles -->
<!-- include the carousel nav here. -->
</main><!-- #main -->
</div><!-- #primary -->
<!-- Do the right sidebar check. -->
<div class="col-md-4 widget-area" id="right-sidebar" role="complementary">
<aside id="fb_related_post_widget-4" class="widget widget_fb_related_post_widget"><h3 class="widget-title">Related Posts</h3><hr>
<div class="aside-related-projects"><div class="article-grid-redesign">
<article class="post-17280 post type-post status-publish has-post-thumbnail hentry" id="post-17280">
<a href="https://engineering.fb.com/2021/02/23/data-infrastructure/silent-data-corruption/" rel="bookmark" aria-label="Read More about Mitigating the effects of silent data corruption at scale">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2021/02/RiB_DarkBlue.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2021/02/RiB_DarkBlue.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2021/02/RiB_DarkBlue.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Feb 23, 2021">Feb 23, 2021</time></span>
<h4 class="entry-title">Mitigating the effects of silent data corruption at scale</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div><div class="article-grid-redesign">
<article class="post-17009 post type-post status-publish has-post-thumbnail hentry" id="post-17009">
<a href="https://engineering.fb.com/2020/12/09/data-center-engineering/how-facebook-keeps-its-large-scale-infrastructure-hardware-up-and-running/" rel="bookmark" aria-label="Read More about How Facebook keeps its large-scale infrastructure hardware up and running">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2020/12/Prineville_Server-room-dark2-e1607617109406.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2020/12/Prineville_Server-room-dark2-e1607617109406.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2020/12/Prineville_Server-room-dark2-e1607617109406.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Dec 09, 2020">Dec 09, 2020</time></span>
<h4 class="entry-title">How Facebook keeps its large-scale infrastructure hardware up and running</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div><div class="article-grid-redesign">
<article class="post-18420 post type-post status-publish has-post-thumbnail hentry" id="post-18420">
<a href="https://engineering.fb.com/2021/12/16/data-center-engineering/power-loss-siren/" rel="bookmark" aria-label="Read More about Power Loss Siren: Making Meta resilient to power loss events">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2021/12/Eng-Blog-Self-Serve-Hero-Images-DATACENTERS-501-Teale.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2021/12/Eng-Blog-Self-Serve-Hero-Images-DATACENTERS-501-Teale.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2021/12/Eng-Blog-Self-Serve-Hero-Images-DATACENTERS-501-Teale.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Dec 16, 2021">Dec 16, 2021</time></span>
<h4 class="entry-title">Power Loss Siren: Making Meta resilient to power loss events</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div></div></aside><aside id="wpb_widget-3" class="widget widget_wpb_widget"><h3 class="widget-title">Related Positions</h3><hr><ul class="available-positions-list"> </ul>
<a href="https://www.metacareers.com">
<div class="btn available-positions-btn">
See All Jobs </div>
</a>
</aside>
</div><!-- #secondary -->
<!-- <php get_template_part('content-single-side-bar'); ?> -->
<!-- Do the right sidebar check. -->
</div><!-- Container end -->
</div><!-- Wrapper end -->
</div>
<!-- ******************* The Footer Full-width Widget Area ******************* -->
<div class="wrapper wrapper-community-outer">
<div class="wrapper-community">
<div class="community" tabindex="-1">
<div class="row">
<!-- community section is generated through WordPress widget -->
<div id="wpb_widget-2" class="footer-widget widget_wpb_widget widget-count-3 col-md-4"><h3 class="widget-title">Available Positions</h3><hr><ul class="available-positions-list"> </ul>
<a href="https://www.metacareers.com">
<div class="btn available-positions-btn">
See All Jobs </div>
</a>
</div><!-- .footer-widget --><div id="custom_html-2" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Technology at Meta</h3><div class="textwidget custom-html-widget"><ul class="stay-connected-lists">
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-fb-engineering" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Engineering at Meta - X
</div>
<a onclick='return twitterFollow("Meta_Engineers")'>
<button class="btn stay-connected-btn">
Follow
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-AI" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
AI at Meta
</div>
<a href="https://ai.meta.com/blog/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta Quest Blog
</div>
<a href="https://www.meta.com/blog/quest/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta for Developers
</div>
<a href="https://developers.facebook.com/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-bug-bounty" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta Bug Bounty
</div>
<a href="https://bugbounty.meta.com/">
<button class="btn stay-connected-btn">
Learn more
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/rss.png" alt="footer-rss" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
RSS
</div>
<a href="https://code.facebook.com/posts/rss/">
<button class="btn stay-connected-btn">
Subscribe
</button>
</a>
</div>
</li>
</ul>
</div></div><!-- .footer-widget --><div id="custom_html-3" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Open Source</h3><div class="textwidget custom-html-widget"><p class="opensource-desc">
Meta believes in building community through open source technology. Explore our latest projects in Artificial Intelligence, Data Infrastructure, Development Tools, Front End, Languages, Platforms, Security, Virtual Reality, and more.
</p>
<ul class="opensource-cats">
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/android.png" alt="android" loading="lazy" width="50" height="59">
<div class="opensource-cap">
ANDROID
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/ios.png" alt="ios" loading="lazy" width="48" height="58">
<div class="opensource-cap">
iOS
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/web.png" alt="web" loading="lazy" width="55" height="49">
<div class="opensource-cap">
WEB
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/backend.png" alt="backend" loading="lazy" width="56" height="51">
<div class="opensource-cap">
BACKEND
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/hardware.png" alt="hardware" loading="lazy" width="55" height="44">
<div class="opensource-cap">
HARDWARE
</div>
</li>
</ul>
<button class="btn opensource-btn" onclick="window.location.href='https://opensource.fb.com/'">
Learn More
</button>
</div></div><!-- .footer-widget -->
</div>
</div>
</div>
</div><!-- #wrapper-footer-full -->
<div class="wrapper-footer" id="wrapper-footer">
<div class="inner-container">
<div class="footer">
<div class="row">
<div class="col-md-12">
<footer class="meta-footer" id="meta-footer">
<div class="site-info row justify-content-between">
<div class="col-md-5 meta-logo-container">
<a href="https://about.facebook.com/">
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/meta_logo_full.svg" width="89" height="18" alt="Meta" />
</a>
<div id="text-2" class="widget widget_text"> <div class="textwidget"><p>Engineering at Meta is a technical news resource for engineers interested in how we solve large-scale technical challenges at Meta.</p>
</div>
</div> </div>
<div class="col-md-6 footer-meta-links-container">
<ul class="meta-footer-links">
<li class="footer-link"><a href="https://engineering.fb.com">Home</a></li><li class="footer-link"><a href="https://about.meta.com/">Company Info</a></li><li class="footer-link"><a href="https://www.metacareers.com/?ref=engineering.fb.com">Careers</a></li> </ul>
</div>
</div>
</footer>
</div>
</div>
</div>
</div>
<div class="footer">
<div class="row">
<div class="col-md-12">
<footer class="site-footer" id="colophon">
<div class="inner-container">
<div class="site-info row justify-content-between">
<div class="col-md-4 copyright-container">
<div class="copyright"> © 2026 Meta</div>
</div>
<!-- static footer -->
<div class="col-md-7 footer-links-container">
<ul class="footer-links">
<li class="footer-link"><a href="https://www.facebook.com/policies">Terms</a></li><li class="footer-link"><a href="https://www.facebook.com/privacy/policy">Privacy</a></li><li class="footer-link"><a href="/privacy">Cookies</a></li><li class="footer-link"><a href="https://www.facebook.com/help">Help</a></li> </ul>
</div>
</div>
</div><!-- .site-info -->
</footer><!-- #colophon -->
</div><!--col end -->
</div><!-- row end -->
</div><!-- //.inner-container -->
</div><!-- wrapper end -->
</div><!-- #page we need this extra closing tag here -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/code-fb-com/*","/wp-content/themes/understrap/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<div class="gdprconsent-container" id="GDPRConsentBar">
<div class="gdprconsent-wrapper">
<div class="gdprconsent-content">
<p>To help personalize content, tailor and measure ads and provide a safer experience, we use cookies. By clicking or navigating the site, you agree to allow our collection of information on and off Facebook through cookies. Learn more, including about available controls: <a href="/privacy">Cookie Policy</a></p>
<div class="gdprconsent-button">
<button type="button" data-gdpr-accept>Accept</button>
</div>
</div>
</div>
</div>
<!-- Twitter universal website tag code -->
<script>
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
// Insert Twitter Pixel ID and Standard Event data below
twq('init','o31u0');
twq('track','PageView');
});
}
</script>
<!-- End Twitter universal website tag code -->
<script type="text/javascript">
window.WPCOM_sharing_counts = {"https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/":18566};
</script>
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KZRLGZ8" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=660742068067366&ev=PageView&noscript=1"/></noscript>
<script id="child-understrap-scripts-js-extra">
var loadmore_params = {"restfulURL":"https://engineering.fb.com/wp-json/fb/v1/loadmore","searchText":"","posts":"{\"page\":0,\"year\":2022,\"monthnum\":3,\"day\":17,\"name\":\"silent-errors\",\"category_name\":\"production-engineering\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","post_type":"post"};
var search_params = {"ajaxurl":"https://engineering.fb.com/wp-admin/admin-ajax.php","posts":"{\"page\":0,\"year\":2022,\"monthnum\":3,\"day\":17,\"name\":\"silent-errors\",\"category_name\":\"production-engineering\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","article_filter":"2f23ad5ce3"};
var more_articles = {"next_article":{"label":"Next","permalink":"https://engineering.fb.com/2022/03/30/security/de-identified-authentication-at-scale/","title":"How Meta enables de-identified authentication at scale","thumbnail":["https://engineering.fb.com/wp-content/uploads/2022/03/CD22_138-Eng-Blog-De-Identified-Authorization-Hero.jpg?w=580&h=326&crop=1",580,326,true]},"prev_article":{"label":"Prev","permalink":"https://engineering.fb.com/2022/03/15/developer-tools/vespa/","title":"VESPA: Static profiling for binary optimization","thumbnail":["https://engineering.fb.com/wp-content/uploads/2021/08/RiB_RichTeal.jpg?w=580&h=326&crop=1",580,326,true]}};
//# sourceURL=child-understrap-scripts-js-extra
</script>
<script type="text/javascript" src="https://engineering.fb.com/_static/??/wp-content/themes/code-fb-com/dist/js/child-theme.min.js,/wp-content/themes/code-fb-com/js/prism.js?m=1783017802j" ></script><script id="sharing-js-js-extra">
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":""};
//# sourceURL=sharing-js-js-extra
</script>
<script id="sharing-js-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/_inc/build/sharedaddy/sharing.min.js?ver=16.1.3"></script>
<script id="sharing-js-js-after">
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-facebook' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-threads' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-threads' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomthreads', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-linkedin' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-x' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-x' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomx', 'menubar=1,resizable=1,width=600,height=350' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-bluesky' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-bluesky' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcombluesky', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-mastodon' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-mastodon' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcommastodon', 'menubar=1,resizable=1,width=460,height=400' );
return false;
}
} );
} )();
//# sourceURL=sharing-js-js-after
</script>
<script id="video-analytics-js-after">
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
let youtube = document.querySelectorAll('[src*="youtube-nocookie"]');
for (let i = 0; i < youtube.length; i++) {
youtube[i].src = youtube[i].src.replace('youtube-nocookie.com', 'youtube.com');
}
let vimeo = document.querySelectorAll('[src*="vimeo"]');
for (let i = 0; i < vimeo.length; i++) {
vimeo[i].src = vimeo[i].src.replace('&dnt=true', '');
}
});
}
//# sourceURL=video-analytics-js-after
</script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://engineering.fb.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://engineering.fb.com/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>