1223 lines
119 KiB
HTML
1223 lines
119 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en-US">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||
<meta name="mobile-web-app-capable" content="yes">
|
||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||
<meta name="apple-mobile-web-app-title" content="Engineering at Meta - Engineering at Meta Blog">
|
||
<meta property="fb:app_id" content="1425766027653270" />
|
||
<link rel="profile" href="http://gmpg.org/xfn/11">
|
||
<link rel="pingback" href="https://engineering.fb.com/xmlrpc.php">
|
||
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
|
||
<script async src="https://engineering.fb.com/wp-content/themes/code-fb-com/dist/js/optimize.min.js"></script>
|
||
|
||
<!-- This site is optimized with the Yoast SEO Premium plugin v19.3 (Yoast SEO v27.3) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
|
||
<title>How Meta keeps its AI hardware reliable - Engineering at Meta</title>
|
||
<link rel="canonical" href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/" />
|
||
<meta name="twitter:label1" content="Written by" />
|
||
<meta name="twitter:data1" content="Harish Dattatraya Dixit, Sriram Sankar" />
|
||
<meta name="twitter:label2" content="Est. reading time" />
|
||
<meta name="twitter:data2" content="15 minutes" />
|
||
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/#article","isPartOf":{"@id":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/"},"author":[{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/d9be7d06892b1301d9d6aab41ecca8a5"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/32a3a81f2a9f159cedb1ec17f50dca2e"}],"headline":"How Meta keeps its AI hardware reliable","datePublished":"2025-07-22T18:45:16+00:00","mainEntityOfPage":{"@id":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/"},"wordCount":2923,"publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"image":{"@id":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2025\/07\/Meta-AI-Hardware-Reliability-at-Scale-HERO.png","articleSection":["Data Infrastructure"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/","url":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/","name":"How Meta keeps its AI hardware reliable - Engineering at Meta","isPartOf":{"@id":"https:\/\/engineering.fb.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/#primaryimage"},"image":{"@id":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2025\/07\/Meta-AI-Hardware-Reliability-at-Scale-HERO.png","datePublished":"2025-07-22T18:45:16+00:00","breadcrumb":{"@id":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/#primaryimage","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2025\/07\/Meta-AI-Hardware-Reliability-at-Scale-HERO.png","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2025\/07\/Meta-AI-Hardware-Reliability-at-Scale-HERO.png","width":1280,"height":720},{"@type":"BreadcrumbList","@id":"https:\/\/engineering.fb.com\/2025\/07\/22\/data-infrastructure\/how-meta-keeps-its-ai-hardware-reliable\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/engineering.fb.com\/"},{"@type":"ListItem","position":2,"name":"How Meta keeps its AI hardware reliable"}]},{"@type":"WebSite","@id":"https:\/\/engineering.fb.com\/#website","url":"https:\/\/engineering.fb.com\/","name":"Engineering at Meta","description":"Engineering at Meta Blog","publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/engineering.fb.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/engineering.fb.com\/#organization","name":"Meta","url":"https:\/\/engineering.fb.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","width":29011,"height":12501,"caption":"Meta"},"image":{"@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/Engineering\/","https:\/\/x.com\/fb_engineering"]},[],{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/d9be7d06892b1301d9d6aab41ecca8a5","name":"Harish Dattatraya Dixit","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/d4f78ec81fa9cf745ea35e273a1c3a085ab1347fec2cc60beb9f4810d1779719?s=96&d=mm&r=gfdb9447c64fd426f9f26001feed6d21a","url":"https:\/\/secure.gravatar.com\/avatar\/d4f78ec81fa9cf745ea35e273a1c3a085ab1347fec2cc60beb9f4810d1779719?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/d4f78ec81fa9cf745ea35e273a1c3a085ab1347fec2cc60beb9f4810d1779719?s=96&d=mm&r=g","caption":"Harish Dattatraya Dixit"},"url":"https:\/\/engineering.fb.com\/author\/harish-dattatraya-dixit\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/32a3a81f2a9f159cedb1ec17f50dca2e","name":"Sriram Sankar","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Sriram Sankar"},"url":"https:\/\/engineering.fb.com\/author\/sriram-sankar\/"}]}</script>
|
||
<!-- / Yoast SEO Premium plugin. -->
|
||
|
||
|
||
<link rel='dns-prefetch' href='//secure.gravatar.com' />
|
||
<link rel='dns-prefetch' href='//v0.wordpress.com' />
|
||
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta » Feed" href="https://engineering.fb.com/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta » Comments Feed" href="https://engineering.fb.com/comments/feed/" />
|
||
<script async src="https://www.googleoptimize.com/optimize.js?id=OPT-TPNDQC5" onerror="dataLayer.hide.end && dataLayer.hide.end()"></script>
|
||
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2025%2F07%2F22%2Fdata-infrastructure%2Fhow-meta-keeps-its-ai-hardware-reliable%2F" />
|
||
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2025%2F07%2F22%2Fdata-infrastructure%2Fhow-meta-keeps-its-ai-hardware-reliable%2F&format=xml" />
|
||
<style id="wp-img-auto-sizes-contain-inline-css">
|
||
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
|
||
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
|
||
</style>
|
||
<style id="wp-emoji-styles-inline-css">
|
||
|
||
img.wp-smiley, img.emoji {
|
||
display: inline !important;
|
||
border: none !important;
|
||
box-shadow: none !important;
|
||
height: 1em !important;
|
||
width: 1em !important;
|
||
margin: 0 0.07em !important;
|
||
vertical-align: -0.1em !important;
|
||
background: none !important;
|
||
padding: 0 !important;
|
||
}
|
||
/*# sourceURL=wp-emoji-styles-inline-css */
|
||
</style>
|
||
<style id="wp-block-library-inline-css">
|
||
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
|
||
|
||
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
|
||
</style>
|
||
<style id="classic-theme-styles-inline-css">
|
||
/*! This file is auto-generated */
|
||
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
|
||
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
|
||
</style>
|
||
|
||
<style id="global-styles-inline-css">
|
||
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
|
||
/*# sourceURL=global-styles-inline-css */
|
||
</style>
|
||
|
||
<link rel='stylesheet' id='all-css-12' href='https://engineering.fb.com/_static/??-eJyNjUsOwjAMRC9EaiqghQXiKFVqhzSQn+pEiNuTFJBAbNjZTzNv4BYFBp+UTxBt1sYzFKQpzpVz5WQ4QSUv0CDzCj6KaVJOMWAgJc5jwe7ZKTnAyVgSS6Jxxv90XRbv3YtKUeJVtF3TgguUbZHyJGdFkui+nMbrvxWD8QhjLvvAAY20wgYd+OupspM7tv3+sO533Wb7AImQY4U=' type='text/css' media='all' />
|
||
<script id="gdprconsent-js" src="https://engineering.fb.com/wp-content/plugins/wp-gdpr-consent/dist/gdprconsent.js?ver=2.0.0"></script>
|
||
<script id="gdprconsent-js-after">
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack('https://www.googletagmanager.com/gtag/js?id=391063774');
|
||
window.gdprSafeTrack(function() {
|
||
var google_analytics_g4 = decodeURIComponent( '391063774' );
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
gtag('js', new Date());
|
||
gtag('config', google_analytics_g4);
|
||
});
|
||
}
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
var google_tag_manager = decodeURIComponent( 'GTM-KZRLGZ8' );
|
||
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||
})(window,document,'script','dataLayer',google_tag_manager);
|
||
});
|
||
}
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
var facebook_pixel = decodeURIComponent( '660742068067366' );
|
||
!function(f,b,e,v,n,t,s)
|
||
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
|
||
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
|
||
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
|
||
n.queue=[];t=b.createElement(e);t.async=!0;
|
||
t.src=v;s=b.getElementsByTagName(e)[0];
|
||
s.parentNode.insertBefore(t,s)}(window, document,'script',
|
||
'https://connect.facebook.net/en_US/fbevents.js');
|
||
fbq('init', facebook_pixel);
|
||
fbq('track', 'PageView');
|
||
});
|
||
}
|
||
//# sourceURL=gdprconsent-js-after
|
||
</script>
|
||
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJzTLy/QzcxLzilNSS3WzwKiwtLUokoopZebmaeXVayjj0+Rbm5melFiSSpUsX2uraG5hbmZgYmloUkWAK+3Iic=" ></script><script type="text/javascript" src="https://engineering.fb.com/wp-content/themes/understrap/js/popper.min.js?m=1773259670g" ></script><link rel="https://api.w.org/" href="https://engineering.fb.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.fb.com/wp-json/wp/v2/posts/22663" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.fb.com/xmlrpc.php?rsd" />
|
||
<meta name="generator" content="WordPress 7.1" />
|
||
<link rel='shortlink' href='https://wp.me/pa0Lhq-5Tx' />
|
||
<link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/Optimistic_Display_W_Md.woff2" as="font" type="font/woff2" crossorigin="anonymous">
|
||
<!-- Jetpack Open Graph Tags -->
|
||
<meta property="og:type" content="article" />
|
||
<meta property="og:title" content="How Meta keeps its AI hardware reliable" />
|
||
<meta property="og:url" content="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/" />
|
||
<meta property="og:description" content="Hardware faults can have a significant impact on AI training and inference. Silent data corruptions (SDCs), undetected data errors caused by hardware, can be particularly harmful for AI systems tha…" />
|
||
<meta property="article:published_time" content="2025-07-22T18:45:16+00:00" />
|
||
<meta property="article:modified_time" content="2025-07-22T18:25:19+00:00" />
|
||
<meta property="og:site_name" content="Engineering at Meta" />
|
||
<meta property="og:image" content="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-Hardware-Reliability-at-Scale-HERO.png" />
|
||
<meta property="og:image:width" content="1280" />
|
||
<meta property="og:image:height" content="720" />
|
||
<meta property="og:image:alt" content="" />
|
||
<meta property="og:locale" content="en_US" />
|
||
<meta name="twitter:text:title" content="How Meta keeps its AI hardware reliable" />
|
||
<meta name="twitter:image" content="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-Hardware-Reliability-at-Scale-HERO.png?w=640" />
|
||
<meta name="twitter:card" content="summary_large_image" />
|
||
|
||
<!-- End Jetpack Open Graph Tags -->
|
||
<style id="wp-custom-css">
|
||
.table-of-contents {
|
||
position: sticky;
|
||
top: 20px;
|
||
background: #fff;
|
||
padding: 1em;
|
||
border: 1px solid #ddd;
|
||
border-radius: 8px;
|
||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||
max-width: 250px;
|
||
}
|
||
</style>
|
||
|
||
<link rel="shortcut icon" href="https://engineering.fb.com/wp-content/themes/code-fb-com/favicon.ico" />
|
||
<link rel='stylesheet' id='all-css-0' href='https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/jetpack_vendor/automattic/jetpack-classic-theme-helper/dist/responsive-videos/responsive-videos.css?m=1789075632g' type='text/css' media='all' />
|
||
|
||
</head>
|
||
|
||
<body class="wp-singular post-template-default single single-post postid-22663 single-format-standard wp-theme-understrap wp-child-theme-code-fb-com group-blog">
|
||
|
||
<div class="hfeed site" id="page">
|
||
|
||
<!-- ******************* The Navbar Area ******************* -->
|
||
<div class="wrapper-fluid wrapper-navbar" id="wrapper-navbar">
|
||
<a class="skip-link screen-reader-text sr-only" href="#content">
|
||
Skip to content </a>
|
||
|
||
<nav class="navbar navbar-expand-std">
|
||
|
||
<div class="nav-container">
|
||
|
||
|
||
<!-- Your site title as branding in the menu -->
|
||
|
||
<div class="nav-top-row">
|
||
|
||
<h1 class="navbar-brand mb-0">
|
||
<a rel="home" href="https://engineering.fb.com/" title="Engineering at Meta">
|
||
<img class="navbar-brand-logo" src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/logo-meta.svg" class="navbar-brand-logo" alt="Engineering at Meta" width="215" height="40" loading="lazy">
|
||
</a>
|
||
</h1>
|
||
|
||
<!-- end custom logo -->
|
||
<label for="search" class="screen-reader-text">Search this site</label>
|
||
<input type="text" id="search" name="search" class="form-control-lg navbar-search" autocomplete="off">
|
||
|
||
<button class="navbar-toggler navbar-searchbar-toggler"
|
||
type="button"
|
||
data-toggle="collapse"
|
||
data-target="#searchbarDropdown"
|
||
aria-controls="searchbarDropdown"
|
||
aria-expanded="false"
|
||
aria-label="Toggle navigation">
|
||
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/icon-search.svg" class="navbar-mobile-search-icon" loading="lazy" width="24" height="25" />
|
||
</button>
|
||
|
||
<!-- desktop language menu (mobile language menu is implemented via jQuery)-->
|
||
<div class="notranslate nice-select navbar-language form-control">
|
||
</div>
|
||
|
||
<button class="navbar-toggler collapsed navbar-menu-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
|
||
</div>
|
||
|
||
<div class="navbar-border"></div>
|
||
|
||
<!-- The WordPress Menu goes here -->
|
||
<div id="navbarNavDropdown" class="collapse navbar-collapse"><ul id="main-menu" class="navbar-nav"><li id="menu-item-60" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-60 dropdown"><a title="Open Source" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Open Source <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-12330" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-12330"><a title="Open Source" href="https://engineering.fb.com/category/open-source/" class="nav-link">Open Source</a></li>
|
||
<li id="menu-item-12331" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-12331"><a title="Meta Open Source" href="https://opensource.fb.com" class="nav-link">Meta Open Source</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2686" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2686 dropdown"><a title="Platforms" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Platforms <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-65" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-65"><a title="Android" href="https://engineering.fb.com/category/android/" class="nav-link">Android</a></li>
|
||
<li id="menu-item-66" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-66"><a title="iOS" href="https://engineering.fb.com/category/ios/" class="nav-link">iOS</a></li>
|
||
<li id="menu-item-67" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-67"><a title="Web" href="https://engineering.fb.com/category/web/" class="nav-link">Web</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2682" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2682 dropdown"><a title="Infrastructure Systems" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Infrastructure Systems <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-218" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-218"><a title="Core Infra" href="https://engineering.fb.com/category/core-infra/" class="nav-link">Core Infra</a></li>
|
||
<li id="menu-item-219" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-219"><a title="Data Infrastructure" href="https://engineering.fb.com/category/data-infrastructure/" class="nav-link">Data Infrastructure</a></li>
|
||
<li id="menu-item-220" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-220"><a title="DevInfra" href="https://engineering.fb.com/category/developer-tools/" class="nav-link">DevInfra</a></li>
|
||
<li id="menu-item-226" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-226"><a title="Production Engineering" href="https://engineering.fb.com/category/production-engineering/" class="nav-link">Production Engineering</a></li>
|
||
<li id="menu-item-227" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-227"><a title="Security & Privacy" href="https://engineering.fb.com/category/security/" class="nav-link">Security & Privacy</a></li>
|
||
<li id="menu-item-18607" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18607"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/systems-infrastructure/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2683" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2683 dropdown"><a title="Physical Infrastructure" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Physical Infrastructure <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-215" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-215"><a title="Connectivity" href="https://engineering.fb.com/category/connectivity/" class="nav-link">Connectivity</a></li>
|
||
<li id="menu-item-216" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-216"><a title="Data Center Engineering" href="https://engineering.fb.com/category/data-center-engineering/" class="nav-link">Data Center Engineering</a></li>
|
||
<li id="menu-item-217" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-217"><a title="Networking & Traffic" href="https://engineering.fb.com/category/networking-traffic/" class="nav-link">Networking & Traffic</a></li>
|
||
<li id="menu-item-18608" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18608"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/networking-connectivity/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2684" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2684 dropdown"><a title="Video Engineering & AR/VR" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Video Engineering & AR/VR <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-225" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-225"><a title="Video Engineering" href="https://engineering.fb.com/category/video-engineering/" class="nav-link">Video Engineering</a></li>
|
||
<li id="menu-item-223" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-223"><a title="Virtual Reality" href="https://engineering.fb.com/category/virtual-reality/" class="nav-link">Virtual Reality</a></li>
|
||
<li id="menu-item-13388" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13388"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/augmented-reality-virtual-reality/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2685" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2685 dropdown"><a title="Artificial Intelligence" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Artificial Intelligence <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-222" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-222"><a title="ML Applications" href="https://engineering.fb.com/category/ml-applications/" class="nav-link">ML Applications</a></li>
|
||
<li id="menu-item-221" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-221"><a title="AI Research" href="https://engineering.fb.com/category/ai-research/" class="nav-link">AI Research</a></li>
|
||
<li id="menu-item-13387" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13387"><a title="Research Publications" href="https://ai.facebook.com/results/?content_types%5B0%5D=publication" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-228" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-228"><a title="Watch Videos" href="/videos" class="nav-link">Watch Videos</a></li>
|
||
</ul></div> </div><!-- .container -->
|
||
|
||
<div class="search-container collapse" id="searchbarDropdown">
|
||
<div class="searchbar-search">
|
||
<div class="searchbar-container">
|
||
|
||
<div class="navbar-mobile-search-margin"></div>
|
||
<input type="text" name="search" placeholder="Search" class="form-control-lg navbar-mobile-search" autofocus autocomplete="off">
|
||
<button class="searchbar-clear">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</nav><!-- .site-navigation -->
|
||
|
||
</div><!-- .wrapper-navbar end -->
|
||
|
||
<div class="wrapper" id="single-wrapper">
|
||
|
||
<div class="single-container row" id="content" tabindex="-1">
|
||
|
||
<!-- Do the left sidebar check. -->
|
||
|
||
|
||
|
||
<div class="content-area" id="primary">
|
||
<main class="site-main" id="main">
|
||
|
||
|
||
|
||
<article class="post-22663 post type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure fb_content_type-article" id="post-22663">
|
||
|
||
<header class="entry-header">
|
||
<div class="entry-meta">
|
||
POSTED ON <span class="posted-on entry-date"><time class="published updated" datetime="2025-07-22">JULY 22, 2025</time></span> TO <span class="cat-links"><a href="https://engineering.fb.com/category/data-infrastructure/" rel="category tag">Data Infrastructure</a></span>
|
||
<h1 class="entry-title"><div>How Meta keeps its AI hardware reliable</div></h1> </div><!-- .entry-meta -->
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
<figure id="post-feat-image-container">
|
||
<img width="1280" height="720" src="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-Hardware-Reliability-at-Scale-HERO.png" class="wp-post-image" alt="" /> </figure>
|
||
|
||
<div class="entry-authors-social">
|
||
<div class="entry-authors">
|
||
By <a href="https://engineering.fb.com/author/harish-dattatraya-dixit/" title="Posts by Harish Dattatraya Dixit" class="author url fn" rel="author">Harish Dattatraya Dixit</a>, <a href="https://engineering.fb.com/author/sriram-sankar/" title="Posts by Sriram Sankar" class="author url fn" rel="author">Sriram Sankar</a> </div>
|
||
</div>
|
||
|
||
|
||
<div class="entry-content">
|
||
|
||
<h2 id="talk"></h2>
|
||
<ul>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Hardware faults can have a significant impact on AI training and inference.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Silent data corruptions (SDCs), undetected data errors caused by hardware, can be particularly harmful for AI systems that rely on accurate data for training as well as providing useful outputs.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">We are sharing methodologies we deploy at various scales for detecting SDC across our AI and non-AI infrastructure to help ensure the reliability of AI training and inference workloads across Meta.</span></li>
|
||
</ul>
|
||
<div class="jetpack-video-wrapper"><span class="embed-youtube" style="text-align:center; display: block;"><iframe class="youtube-player" width="4000" height="2250" src="https://www.youtube.com/embed/4EZhnYwcPwQ?version=3&rel=1&showsearch=0&showinfo=1&iv_load_policy=1&fs=1&hl=en-US&autohide=2&wmode=transparent" allowfullscreen="true" style="border:0;" sandbox="allow-scripts allow-same-origin allow-popups allow-presentation allow-popups-to-escape-sandbox"></iframe></span></div>
|
||
<p><span style="font-weight: 400;">Meta’s global AI infrastructure consists of a large number of hardware components and servers , connected via network fabric across globally distributed data centers. This setup integrates storage, compute, and network architectures with unique file systems and PyTorch applications tailored for training or inference workloads. This infrastructure supports training</span> large-scale models <span style="font-weight: 400;">as well as advanced AI applications such as text-to-image generation and </span><a href="https://ai.meta.com/blog/instagram-edits-cutouts-segment-anything/" target="_blank" rel="noopener"><span style="font-weight: 400;">object segmentation</span></a><span style="font-weight: 400;">. </span></p>
|
||
<p><span style="font-weight: 400;">Since 2018, Meta’s</span> <a href="https://ieeexplore.ieee.org/abstract/document/8416200/" target="_blank" rel="noopener"><span style="font-weight: 400;">hardware reliability journey</span></a><span style="font-weight: 400;"> has led to</span> <a href="https://dl.acm.org/doi/abs/10.1145/3358960.3375793" target="_blank" rel="noopener"><span style="font-weight: 400;">novel findings</span></a><span style="font-weight: 400;">, identifying unique failure types in disks, CPUs, memories, switches, GPUs, ASICs, and networks, often leading the industry in discovering failure modes. We have developed mitigation policies to ensure smooth infrastructure operation and availability for billions of users and thousands of internal use cases. </span><a href="https://engineering.fb.com/2024/03/12/data-center-engineering/building-metas-genai-infrastructure/" target="_blank" rel="noopener"><span style="font-weight: 400;">As we continue to build large AI clusters</span></a><span style="font-weight: 400;">, understanding hardware failures and mitigation strategies is crucial for the reliable training of large-scale AI models.</span></p>
|
||
<p><span style="font-weight: 400;">Training large-scale models involves thousands of accelerators in a synchronous environment, where any component failure can interrupt or halt the process. We focus on reducing hardware failures during training through detection and diagnostics, and quickly restarting training with healthy servers and accelerators. This involves optimizing fault categorization, device triage, node selection, cluster validation, and checkpoint restore.</span></p>
|
||
<p><span style="font-weight: 400;">From our experience running </span><a href="https://arxiv.org/abs/2407.21783" target="_blank" rel="noopener"><span style="font-weight: 400;">the Llama 3 herd of models</span></a><span style="font-weight: 400;">, we find that hardware failures in components such as SRAMs, HBMs, processing grids, and network switch hardware significantly impact AI cluster reliability, with over 66% of training interruptions due to such failures. Some of the challenges for AI clusters include accelerators that might be less reliable than CPUs due to complexity and limited telemetry, network complexity that could result in misattributed failures, and errors within the GPU software stack that may require extensive configuration to correct. Hence, reducing hardware and configuration failures greatly enhances cluster efficiency.</span></p>
|
||
<p><img decoding="async" class="alignnone size-full wp-image-22664" src="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-1.png" alt="" width="1655" height="926" srcset="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-1.png 1655w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-1.png?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-1.png?resize=916,513 916w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-1.png?resize=768,430 768w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-1.png?resize=1024,573 1024w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-1.png?resize=1536,859 1536w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-1.png?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-1.png?resize=192,107 192w" sizes="(max-width: 992px) 100vw, 62vw" /></p>
|
||
<h2><span style="font-weight: 400;">Types of hardware faults encountered at Meta</span></h2>
|
||
<p><span style="font-weight: 400;">The hardware faults or errors that we observe in our infrastructure can be classified broadly into three categories: </span></p>
|
||
<h3><span style="font-weight: 400;">Static errors </span></h3>
|
||
<p><span style="font-weight: 400;">Hardware failures often appear as binary states: A device either powers on or powers off. These static errors are straightforward to identify in large-scale fleets. If devices fail to power on or enumerate, simple health checks can verify their presence and configurations. As configurations and device scales grow in large training clusters, these faults occur more frequently but are easier to triage, root-cause, and repair, making them manageable at scale. </span></p>
|
||
<h3><span style="font-weight: 400;">Transient errors </span></h3>
|
||
<p><span style="font-weight: 400;">Transient errors, categorized by their reproducibility, include load-dependent or partially observable faults, such as device issues from thermal runaway or random crashes from uncorrectable errors. Mitigation involves understanding manifestation conditions; and our larger scale aids in triaging and pattern matching, setting traps for these conditions. When triggered, devices are marked for mitigation or repair. Advances in RAS telemetry in hyperscale infrastructure have greatly improved this process. Factors including workload sensitivity, temperature range, frequency, and manufacturing parameters contribute to these errors.</span></p>
|
||
<p><span style="font-weight: 400;">Mitigation can also involve inducing conditions with artificial workloads in non-production stages to make faults more repeatable. Additionally, capturing transient states as “sticky” status values provides telemetry indications for hardware failures. Though less frequent than static faults and harder to detect, Meta’s scale and our significant engineering efforts have made these scenarios detectable.</span></p>
|
||
<h3><span style="font-weight: 400;">Silent errors </span></h3>
|
||
<p><span style="font-weight: 400;">Silent errors or </span><a href="https://engineering.fb.com/2021/02/23/data-infrastructure/silent-data-corruption/" target="_blank" rel="noopener"><span style="font-weight: 400;">silent data corruptions (SDCs)</span></a><span style="font-weight: 400;"> occur when hardware miscomputes without leaving detectable traces, leading applications to consume incorrect results. These errors, often due to silicon defects, can remain unnoticed for long periods unless significant deviations are observed. Detecting them requires extensive engineering and costly telemetry to trace data corruption back to specific devices. These faults significantly impact large-scale services due to the lack of telemetry and continued consumption.</span></p>
|
||
<p><a href="https://arxiv.org/abs/2102.11245" target="_blank" rel="noopener"><span style="font-weight: 400;">Case studies</span></a><span style="font-weight: 400;">, including one where a single computation error led to missing rows in a Spark application, highlight the prevalence of silent errors in hyperscale infrastructures. Historically, soft-error-related bitflips were reduced to one fault per million devices, but with increased silicon density in accelerators, silent data corruptions now occur at about one fault per thousand devices, much higher than cosmic-ray-induced soft errors.</span></p>
|
||
<h2><span style="font-weight: 400;">Key challenges presented by SDCs </span></h2>
|
||
<p><span style="font-weight: 400;">SDCs present significant challenges in hyperscale infrastructure due to their data dependency, creating an impractical exponential test space for all possible data values. These faults also depend on device voltage, frequency, operating temperature, and life cycle. For instance, a device may fail computational checks only after months of use, indicating a state of “wear out.” Therefore, consistent, periodic, and frequent testing within a random state space is necessary throughout the device’s life cycle to identify these inaccuracies.</span></p>
|
||
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-22665" src="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-2.png" alt="" width="1733" height="975" srcset="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-2.png 1733w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-2.png?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-2.png?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-2.png?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-2.png?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-2.png?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-2.png?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-2.png?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
|
||
<h2><span style="font-weight: 400;">Novel SDC detection mechanisms </span></h2>
|
||
<p><span style="font-weight: 400;">To protect applications from silent data corruption, Meta employs several detection mechanisms, as detailed in the papers, </span><a href="https://arxiv.org/abs/2203.08989" target="_blank" rel="noopener"><span style="font-weight: 400;">“Detecting Silent Errors in the Wild”</span></a><span style="font-weight: 400;"> and </span><a href="https://dl.acm.org/doi/abs/10.1145/3676641.3716258" target="_blank" rel="noopener"><span style="font-weight: 400;">“Hardware Sentinel.”</span></a></p>
|
||
<ol>
|
||
<li style="font-weight: 400;" aria-level="1"><a href="https://engineering.fb.com/2022/03/17/production-engineering/silent-errors/" target="_blank" rel="noopener"><span style="font-weight: 400;">Fleetscanner</span></a><span style="font-weight: 400;">: Fleetscanner captures performance outliers at scale with targeted micro-benchmarks for identifying hardware defects. These benchmarks’ signatures are integrated into telemetry for non-benchmark-based detection. This approach involves running directed tests during maintenance operations such as firmware upgrades and hardware repairs. Tests are scheduled periodically, covering the entire fleet every 45 to 60 days. While it provides dedicated testing on hosts, it may be too slow for some SDCs.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><a href="https://arxiv.org/abs/2203.08989" target="_blank" rel="noopener"><span style="font-weight: 400;">Ripple:</span></a><span style="font-weight: 400;"> Ripple co-locates with workloads, executing tests in milliseconds to seconds, allowing fleet-wide coverage in days. It overlaps test instructions across cores and threads, providing faster detection than Fleetscanner.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><a href="https://dl.acm.org/doi/abs/10.1145/3676641.3716258" target="_blank" rel="noopener"><span style="font-weight: 400;">Hardware Sentinel:</span></a><span style="font-weight: 400;"> This novel, test-and-architecture-agnostic approach evaluates application exceptions in kernel space. It identifies core-based anomalies as silent data corruption without requiring test allocations, operating solely in the analytical plane. Hardware Sentinel outperforms testing-based methods by 41% across architectures, applications, and data centers.</span></li>
|
||
</ol>
|
||
<p><span style="font-weight: 400;">Combined together, these three mechanisms provide one of the best in-fleet coverage at scale, for detecting and protecting our infrastructure against SDCs.</span></p>
|
||
<h2><span style="font-weight: 400;">Silent errors in AI hardware </span></h2>
|
||
<p><span style="font-weight: 400;">The methodologies described above execute across the fleet and are fully productionized at scale, detecting SDCs across AI and non-AI infrastructure. However, AI applications such as training and inference have unique and more challenging implications for SDCs. </span></p>
|
||
<h3><span style="font-weight: 400;">SDCs in training workloads</span></h3>
|
||
<p><span style="font-weight: 400;">SDCs in training workloads lead to incorrect computations, affecting both forward and backward passes. This results in a divergence from the intended training path, impacting training efficacy. While AI training workloads are sometimes considered self-resilient to SDCs, this is true only for a limited subset of SDC manifestations. In most realistic scenarios, self-resilience is inadequate. SDCs persist across iterations, and the quantization of data values in AI training, which increases information per bit, exacerbates the impact of SDCs, continuously increasing divergence rates in training workloads.</span></p>
|
||
<p><span style="font-weight: 400;">Below we present the two most common cases of training divergence due to SDCs.</span></p>
|
||
<h4><span style="font-weight: 400;">Not-a-Number </span><span style="font-weight: 400;">(NaN) propagation </span></h4>
|
||
<p><span style="font-weight: 400;">Not-a-Number (NaN) propagation occurs when an SDC pushes a representable value into an incorrect representation, generating a NaN during training computations. Once a NaN is created, it propagates through subsequent computations, affecting the training iteration, accelerator domain, host domain, and eventually the entire cluster. This widespread NaN contagion can lead to a cluster halt, as the source—often a few specific computations on a single accelerator—may be difficult to trace amidst the cluster’s scale. Identifying and quarantining the offending accelerator and nodes are necessary to resolve the issue.</span></p>
|
||
<h4><span style="font-weight: 400;">Corrupted gradient variance </span></h4>
|
||
<p><span style="font-weight: 400;">Corrupted gradient variance occurs when an SDC affects gradient calculations, leading to gradient explosion, implosion, or local minima. This corruption, while within numeric bounds, is mistakenly treated as correct, affecting the entire cluster in synchronous training. The corrupted values are exchanged as true values, causing the training to appear to progress without actual improvement. Over time, SDCs aggregate, causing major divergences in gradients, potentially trapping the algorithm in local minima or causing gradient explosions or implosions.</span></p>
|
||
<p><span style="font-weight: 400;">Detecting these SDCs is challenging due to their subtlety and the time required to observe their effects, which can take weeks or months. Unlike NaN propagation, these corruptions are harder to trace and rectify, as they don’t trigger NaN traps. Consequently, SDCs can lead to significant unproductive use of computational resources and training iterations. Without detection, the root cause remains elusive, making subsequent training risky until the offending device is identified and isolated.</span></p>
|
||
<h3><span style="font-weight: 400;">SDCs in inference workloads</span></h3>
|
||
<p><span style="font-weight: 400;">In inference applications, SDCs lead to incorrect results, which, due to the scale of operations, affect thousands of inference consumers. Persistent SDCs can directly impact decisions made by systems such as recommendation engines or LLM outputs. These corruptions can bypass policies related to privacy or integrity, as they are not constrained by boundaries. Consequently, inference corruptions significantly reduce the efficacy of models trained with substantial computational resources, making seemingly benign inference use cases problematic at scale.</span></p>
|
||
<h2><span style="font-weight: 400;">Impact of SDCs</span></h2>
|
||
<p><span style="font-weight: 400;">SDCs in training and inference clusters create complex debugging scenarios across thousands of components. </span></p>
|
||
<p><span style="font-weight: 400;">In training, visible faults halt the cluster, but SDCs create an illusion of progress, obscuring the fault source. NaN propagation requires identifying the offending node; otherwise, restarts from checkpoints will eventually fail. Corrupted gradient variance prolongs this illusion until variances aggregate, making restarts ineffective. SDCs thus cause significant computational inefficiency, with a larger temporal impact than visible faults.</span></p>
|
||
<p><span style="font-weight: 400;">In inference, triage involves costly telemetry at each substage. Until the offending node is identified, inference clusters can’t be used, risking repeat corruption. Large deviations are easier to detect with anomaly detectors, but smaller ones require extensive debugging. This process involves hundreds of engineers, halting production use cases, and impacting reliable capacity for serving production.</span></p>
|
||
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-22666" src="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-3.png" alt="" width="1533" height="863" srcset="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-3.png 1533w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-3.png?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-3.png?resize=916,516 916w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-3.png?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-3.png?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-3.png?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-3.png?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
|
||
<h2><span style="font-weight: 400;">Detection of SDCs in AI hardware </span></h2>
|
||
<p><span style="font-weight: 400;">Mitigation strategies that we run in our infrastructure for dealing with SDCs in AI training workloads are classified into infrastructure strategies and stack strategies:</span></p>
|
||
<h3><span style="font-weight: 400;">Infrastructure strategies</span></h3>
|
||
<p><span style="font-weight: 400;">These are applied during operational triage at the cluster level. They focus on managing and mitigating SDCs through the physical and network infrastructure, ensuring that the hardware and system-level components are robust and capable of handling errors effectively. </span></p>
|
||
<h4><span style="font-weight: 400;">Reductive triage</span></h4>
|
||
<p><span style="font-weight: 400;">This strategy involves conducting a binary search with mini-training iterations on progressively smaller cluster sizes to isolate NaN propagation. The goal is to identify a small cluster that replicates the NaN issue, allowing the offending node to be quarantined for further investigation. A reconstituted cluster with new nodes can then resume training from a saved checkpoint. However, this method relies on the ability to reproduce SDCs, which is not always guaranteed due to their dependence on data, electrical, and temperature variations. For corrupted gradient variance, a similar divide-and-triage approach can be used, but the effectiveness varies with training data and cluster size, despite consistent hyperparameter settings.</span></p>
|
||
<h4><span style="font-weight: 400;">Deterministic training</span></h4>
|
||
<p><span style="font-weight: 400;">This approach involves running a known effective model for a few training iterations to ensure there are no NaNs or gradient divergences. It helps verify computational failures that are not data-dependent, as it guarantees correctness for a specific set of values and training inputs.</span></p>
|
||
<h4><span style="font-weight: 400;">Hyper-checkpointing</span></h4>
|
||
<p><span style="font-weight: 400;">This method involves creating checkpoints at increasingly high frequencies to facilitate faster identification and isolation of the corrupting node. It helps maintain training throughput while containing NaN propagation to a specific accelerator or host, thereby speeding up the triage and quarantine process.</span></p>
|
||
<h3><span style="font-weight: 400;">Stack strategies</span></h3>
|
||
<p><span style="font-weight: 400;">These require coordination with the workload and involve adjustments and enhancements at the software-stack level. This includes implementing error detection and correction mechanisms within the application and software layers to handle SDCs more effectively during training processes.</span></p>
|
||
<h4><span style="font-weight: 400;">Gradient clipping</span></h4>
|
||
<p><span style="font-weight: 400;">This strategy involves enforcing gradient clipping within the training workload to limit values within a specified range, thereby mitigating NaN propagation. Computations exceeding this range are clipped, and NaNs can be detected during this step by setting them to a max or min value based on the operand sign. While effective for some NaNs depending on representation format, it may introduce partial errors in certain cases.</span></p>
|
||
<h4><a href="https://ieeexplore.ieee.org/abstract/document/10020972" target="_blank" rel="noopener"><span style="font-weight: 400;">Algorithmic fault tolerance</span></a></h4>
|
||
<p><span style="font-weight: 400;">This robust approach integrates fault tolerance into training algorithms to handle a range of data corruptions, reducing the need for detection and triage. It enhances computational efficiency with minimal overhead, as demonstrated in CPU training. This method requires understanding common defect modes and investing in engineering across the stack, with modified guarantees to training workloads, albeit with some overhead to the overall training footprint.</span></p>
|
||
<h4><span style="font-weight: 400;">Tri-variate computational training architecture</span></h4>
|
||
<p><span style="font-weight: 400;">This approach uses shadow nodes in synchronous training to mitigate SDCs. Training steps are repeated across different nodes at random iterations, ensuring correct progress after verification. If shadow and live nodes differ, training halts, and only those nodes are investigated. The rest continue with new nodes. This method involves multiple shadow-node pools, a random training-node pool, and specified steps from the same checkpoint. It offers robust training but demands significant algorithmic changes and increased data movement and infrastructure overhead.</span></p>
|
||
<h4><a href="https://arxiv.org/abs/2405.01741" target="_blank" rel="noopener"><span style="font-weight: 400;">Parameter vulnerability factors</span></a></h4>
|
||
<p><span style="font-weight: 400;">This approach identifies vulnerable and resilient layers in machine-learning architectures, allowing mapping of vulnerable layers to resilient hardware and resilient layers to unprotected hardware. This dynamic evaluation must scale with architecture evolution. Resilience often incurs costs in area, power, or performance, so PVF enables targeted resilient design, especially for inference.</span></p>
|
||
<h4><a href="https://dl.acm.org/doi/10.1145/3620666.3651349" target="_blank" rel="noopener"><span style="font-weight: 400;">Divergence detection</span></a></h4>
|
||
<p><span style="font-weight: 400;">This mechanism maintains a distribution map for each neuron to detect divergence from typical output distributions, identifying inference corruptions. Though costly, it can be applied at selected sampling rates for large-scale inference. By preserving each neuron’s behavior for specific workloads, divergence helps detect corruptions during execution.</span></p>
|
||
<p><span style="font-weight: 400;">While we have optimized these different methodologies to run effectively in our infrastructure, it should be noted that they offer varying levels of resilience with distinct operating points and engineering/infrastructure overheads. Depending on the scale and intensity of training and inference workloads, orchestrating these strategies effectively can mitigate SDCs’ adverse effects in AI applications.</span></p>
|
||
<h3><span style="font-weight: 400;">Performance faults and unknown unknowns!</span></h3>
|
||
<p><span style="font-weight: 400;">While SDCs are a major challenge at hyperscale, Meta has been developing solutions to detect performance regressions. </span><a href="https://www.usenix.org/conference/osdi24/presentation/chow" target="_blank" rel="noopener"><span style="font-weight: 400;">ServiceLab</span></a><span style="font-weight: 400;">, for example, is a large-scale performance testing platform that helps identify tiny performance regressions at scale. In addition, Fleetscanner has identified hundreds of performance outliers, seen as an emergent fault mode alongside SDCs.</span></p>
|
||
<p><span style="font-weight: 400;">While current mechanisms detect and address static, transient, and silent faults, the full range of hardware fault variants remains partially uncovered. The unknown unknowns require agile solutions across the entire infrastructure and silicon lifecycle, as well as across the hardware-to-software and application stack, to achieve first-class reliability operations.</span></p>
|
||
<h2><span style="font-weight: 400;">A journey towards industry leadership and standardization</span></h2>
|
||
<p><span style="font-weight: 400;">Meta’s journey toward industry leadership in SDC began with identifying frequent fleet issues in 2016, scaling SDC detection in 2018, and implementing detection frameworks by 2019. By 2020, detection mechanisms were integrated into accelerators, and Meta published the paper, </span><a href="https://arxiv.org/abs/2102.11245"><span style="font-weight: 400;">“Silent Data Corruptions at Scale.”</span></a><span style="font-weight: 400;"> In 2022, Meta introduced </span><a href="https://arxiv.org/abs/2203.08989"><span style="font-weight: 400;">“FleetScanner and Ripple”</span></a><span style="font-weight: 400;"> and conducted an </span><a href="https://research.facebook.com/blog/2022/2/engineering-director-sriram-sankar-discusses-metas-first-research-award-opportunity-in-silent-data-corruptions-at-scale"><span style="font-weight: 400;">RFP</span></a><span style="font-weight: 400;"> for </span><a href="https://fburl.com/n5h6hlml"><span style="font-weight: 400;">academic awards</span></a><span style="font-weight: 400;">, funding five winners. </span></p>
|
||
<p><span style="font-weight: 400;">In 2023, Meta collaborated with industry leaders (Google, Microsoft, ARM, AMD, NVIDIA, and Intel) to enhance server resilience, defining test architectures and metrics. A joint </span><a href="https://www.opencompute.org/blog/computings-hidden-menace-the-ocp-takes-action-against-silent-data-corruption-sdc"><span style="font-weight: 400;">RFP</span></a><span style="font-weight: 400;"> with partners from the</span><a href="https://www.opencompute.org/blog/ocps-server-resilience-initiative-sdc-academic-research-awards-announced"><span style="font-weight: 400;"> Open Compute Project selected six winners for cross-domain SDC research</span></a><span style="font-weight: 400;">. By 2024, Meta’s fleet had advanced </span><a href="https://dl.acm.org/doi/abs/10.1145/3676641.3716258"><span style="font-weight: 400;">AI SDC detection methodologies</span></a><span style="font-weight: 400;"> in production, contributing to research through publications, tutorials, and talks at major conferences and forums, addressing at-scale reliability challenges.</span></p>
|
||
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-22667" src="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-4.png" alt="" width="1999" height="1126" srcset="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-4.png 1999w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-4.png?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-4.png?resize=916,516 916w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-4.png?resize=768,433 768w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-4.png?resize=1024,577 1024w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-4.png?resize=1536,865 1536w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-4.png?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-4.png?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
|
||
<h2><span style="font-weight: 400;">The Meta Training and Inference Accelerator </span></h2>
|
||
<p><span style="font-weight: 400;">Meta is on an ambitious journey toward enabling training and inference accelerators under the </span><a href="https://ai.meta.com/blog/next-generation-meta-training-inference-accelerator-AI-MTIA/"><span style="font-weight: 400;">Meta Training and Inference Accelerator (MTIA)</span></a><span style="font-weight: 400;"> family. On this journey, our goal is to utilize all the lessons learned from the fleet and move toward industry-leading, fleet-reliability practices in MTIA architecture and design practices. Using the factory-to-fleet approach, and consistently revisiting our reliability solutions across the stack, our goal is to deliver a best-in-class, reliable-and-performant solution to add to our infrastructure portfolio of AI hardware and to power AI applications at scale. </span></p>
|
||
<h3><span style="font-weight: 400;">Factory to fleet</span></h3>
|
||
<p><span style="font-weight: 400;">To uncover unknowns early, a comprehensive factory-to-fleet view of the silicon life cycle is key. Innovation is needed in all phases, from design to deployment. In design and architecture, revisiting RAS solutions for scale, life-cycle debug hooks, and telemetry architectures can support tools such as Hardware Sentinel, Fleetscanner, and Ripple. During validation and integration, novel yield analysis, manufacturing diagnostics, and fleet-signature-feedback-based detection can prevent faults before shipping. In AI silicon fleets, user-space diagnostics with periodic testing, coverage maps, and control parameters are beneficial. Large-scale analytics like Hardware Sentinel can detect early wear out and data corruption. Robust firmware hooks and debug architecture provide fast feedback to design and architecture amidst fleet-scale issues.</span></p>
|
||
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-22668" src="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-5.png" alt="" width="1326" height="743" srcset="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-5.png 1326w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-5.png?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-5.png?resize=916,513 916w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-5.png?resize=768,430 768w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-5.png?resize=1024,574 1024w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-5.png?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-5.png?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
|
||
<h3><span style="font-weight: 400;">Stack-level resilience</span></h3>
|
||
<p><span style="font-weight: 400;">Factory-to-fleet solutions offer life-cycle resilience for silicon, but resilience must extend beyond silicon to firmware, compilers, kernels, and operating systems. Investments in resilience architectures are needed for correctness-invariant-instruction heterogeneity and enhanced telemetry for exception tracing. Granular firmware-control mechanisms improve telemetry upon fault detection. At the software and application level, techniques like gradient clipping and algorithmic fault tolerance that we called out in this blog, are crucial for security amidst corruptions. Experience with SDCs shows that in-line software resilience and test-agnostic analytical approaches effectively scale for many SDCs with minimal investment, while testing-based approaches are limited to specific instructions.</span></p>
|
||
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-22669" src="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-6.png" alt="" width="1737" height="972" srcset="https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-6.png 1737w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-6.png?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-6.png?resize=916,513 916w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-6.png?resize=768,430 768w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-6.png?resize=1024,573 1024w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-6.png?resize=1536,860 1536w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-6.png?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2025/07/Meta-AI-hardware-reliability-at-scale-image-6.png?resize=192,107 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
|
||
<p><span style="font-weight: 400;">Hardware faults significantly impact AI training and inference production. As cluster sizes and semiconductor complexity grow, fault complexity will exponentially increase. Solutions must involve factory-to-fleet coordination and stack-level resiliency. For AI applications, treating reliability as a primary design consideration is essential.</span></p>
|
||
<h2><span style="font-weight: 400;">Acknowledgments</span></h2>
|
||
<p><i><span style="font-weight: 400;">The authors would like to thank all the cross-functional engineers and teams instrumental in landing these solutions over the years. This blog accompanies the </span></i><a href="#talk"><i><span style="font-weight: 400;">@Scale conference talk</span></i></a><i><span style="font-weight: 400;">, please check out the </span></i><i><span style="font-weight: 400;">talk </span></i><i><span style="font-weight: 400;">for more details. </span></i></p>
|
||
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-facebook"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-facebook-22663"
|
||
class="share-facebook sd-button share-icon"
|
||
href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/?share=facebook"
|
||
target="_blank"
|
||
aria-labelledby="sharing-facebook-22663"
|
||
>
|
||
<span id="sharing-facebook-22663" hidden>Share on Facebook (Opens in new window)</span>
|
||
<span>Facebook</span>
|
||
</a></li><li class="share-threads"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-threads-22663"
|
||
class="share-threads sd-button share-icon"
|
||
href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/?share=threads"
|
||
target="_blank"
|
||
aria-labelledby="sharing-threads-22663"
|
||
>
|
||
<span id="sharing-threads-22663" hidden>Share on Threads (Opens in new window)</span>
|
||
<span>Threads</span>
|
||
</a></li><li class="share-jetpack-whatsapp"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-whatsapp-22663"
|
||
class="share-jetpack-whatsapp sd-button share-icon"
|
||
href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/?share=jetpack-whatsapp"
|
||
target="_blank"
|
||
aria-labelledby="sharing-whatsapp-22663"
|
||
>
|
||
<span id="sharing-whatsapp-22663" hidden>Share on WhatsApp (Opens in new window)</span>
|
||
<span>WhatsApp</span>
|
||
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-linkedin-22663"
|
||
class="share-linkedin sd-button share-icon"
|
||
href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/?share=linkedin"
|
||
target="_blank"
|
||
aria-labelledby="sharing-linkedin-22663"
|
||
>
|
||
<span id="sharing-linkedin-22663" hidden>Share on LinkedIn (Opens in new window)</span>
|
||
<span>LinkedIn</span>
|
||
</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-reddit-22663"
|
||
class="share-reddit sd-button share-icon"
|
||
href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/?share=reddit"
|
||
target="_blank"
|
||
aria-labelledby="sharing-reddit-22663"
|
||
>
|
||
<span id="sharing-reddit-22663" hidden>Share on Reddit (Opens in new window)</span>
|
||
<span>Reddit</span>
|
||
</a></li><li class="share-x"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-x-22663"
|
||
class="share-x sd-button share-icon"
|
||
href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/?share=x"
|
||
target="_blank"
|
||
aria-labelledby="sharing-x-22663"
|
||
>
|
||
<span id="sharing-x-22663" hidden>Share on X (Opens in new window)</span>
|
||
<span>X</span>
|
||
</a></li><li class="share-bluesky"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-bluesky-22663"
|
||
class="share-bluesky sd-button share-icon"
|
||
href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/?share=bluesky"
|
||
target="_blank"
|
||
aria-labelledby="sharing-bluesky-22663"
|
||
>
|
||
<span id="sharing-bluesky-22663" hidden>Share on Bluesky (Opens in new window)</span>
|
||
<span>Bluesky</span>
|
||
</a></li><li class="share-mastodon"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-mastodon-22663"
|
||
class="share-mastodon sd-button share-icon"
|
||
href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/?share=mastodon"
|
||
target="_blank"
|
||
aria-labelledby="sharing-mastodon-22663"
|
||
>
|
||
<span id="sharing-mastodon-22663" hidden>Share on Mastodon (Opens in new window)</span>
|
||
<span>Mastodon</span>
|
||
</a></li><li class="share-custom share-custom-hackernews"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-custom-22663"
|
||
class="share-custom share-custom-hackernews sd-button share-icon"
|
||
href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/?share=custom-1699562127"
|
||
target="_blank"
|
||
aria-labelledby="sharing-custom-22663"
|
||
>
|
||
<span id="sharing-custom-22663" hidden>Share on Hacker News (Opens in new window)</span>
|
||
<span style="background-image:url("https://news.ycombinator.com/y18.svg");">Hacker News</span>
|
||
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-email-22663"
|
||
class="share-email sd-button share-icon"
|
||
href="mailto:?subject=%5BShared%20Post%5D%20How%20Meta%20keeps%20its%20AI%20hardware%20reliable&body=https%3A%2F%2Fengineering.fb.com%2F2025%2F07%2F22%2Fdata-infrastructure%2Fhow-meta-keeps-its-ai-hardware-reliable%2F&share=email"
|
||
target="_blank"
|
||
aria-labelledby="sharing-email-22663"
|
||
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you're having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="cc6c33fd1e" data-email-share-track-url="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/?share=email">
|
||
<span id="sharing-email-22663" hidden>Email a link to a friend (Opens in new window)</span>
|
||
<span>Email</span>
|
||
</a></li><li class="share-end"></li></ul></div></div></div>
|
||
|
||
</div><!-- .entry-content -->
|
||
|
||
|
||
</article><!-- #post-## -->
|
||
|
||
<footer class="entry-footer">
|
||
|
||
|
||
</footer><!-- .entry-footer -->
|
||
|
||
<div class="fb-like"
|
||
data-href="https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/"
|
||
data-width=""
|
||
data-layout="standard"
|
||
data-action="like"
|
||
data-size="small"
|
||
data-share="true">
|
||
</div>
|
||
|
||
|
||
<div class="related-articles">
|
||
<div class="related-articles__header">
|
||
<h3 class="related-articles__header__headline">Read More in Data Infrastructure</h3>
|
||
<a
|
||
aria-label="View all articles in this Data Infrastructure"
|
||
class="related-articles__header__link"
|
||
href="https://engineering.fb.com/category/data-infrastructure/"
|
||
>
|
||
View All </a>
|
||
</div>
|
||
|
||
<div class="article-grids related-articles__grid">
|
||
<div class="row">
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-22663 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure fb_content_type-article" id="post-24399">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/MetaRoCE-HERO-final.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-08-24T11:02:29-07:00">
|
||
AUG 24, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/metaroce-rdma-transport-ai-ethernet/" rel="bookmark" aria-label="Read More about MetaRoCE: A New RDMA Transport Built for AI-Scale Ethernet">
|
||
MetaRoCE: A New RDMA Transport Built for AI-Scale Ethernet </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-22663 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure fb_content_type-article" id="post-24394">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/Meta-MTIA-300-Hero-final.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Meta MTIA 300 - Hero - final" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-08-24T10:45:52-07:00">
|
||
AUG 24, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/mtia-300-meta-training-chip-built-in-nics/" rel="bookmark" aria-label="Read More about MTIA 300: Meta’s First Training Chip with Built-in NICs and Communication-Offloading Engines">
|
||
MTIA 300: Meta’s First Training Chip with Built-in NICs and Communication-Offloading Engines </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-22663 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure fb_content_type-article" id="post-23996">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/05/Meta-Mult-stage-Ads-Ranking.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-08-05T12:20:20-07:00">
|
||
AUG 5, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/08/05/ml-applications/from-user-sequences-to-scaling-laws-a-multi-stage-architecture-for-metas-ads-ranking/" rel="bookmark" aria-label="Read More about From User Sequences to Scaling Laws: A Multi-Stage Architecture for Meta’s Ads Ranking">
|
||
From User Sequences to Scaling Laws: A Multi-Stage Architecture for Meta’s Ads Ranking </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-22663 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure fb_content_type-article" id="post-24167">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/07/Training-GEM-at-LLM-Scale-Hero.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-08-03T11:00:17-07:00">
|
||
AUG 3, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/08/03/ml-applications/training-gem-at-llm-scale-meta-ads-recommendation-foundation-model/" rel="bookmark" aria-label="Read More about GEM Training: How Meta Doubled the Efficiency of Its LLM-Scale Ads Foundation Model">
|
||
GEM Training: How Meta Doubled the Efficiency of Its LLM-Scale Ads Foundation Model </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-22663 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure fb_content_type-article" id="post-24150">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Metas-AI-Storage-Blueprint-at-Scale-Hero-1.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-07-01T09:00:36-07:00">
|
||
JUL 1, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/07/01/data-infrastructure/metas-ai-storage-blueprint-at-scale/" rel="bookmark" aria-label="Read More about Meta’s AI Storage Blueprint at Scale">
|
||
Meta’s AI Storage Blueprint at Scale </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-22663 type-post status-publish format-standard has-post-thumbnail hentry category-data-infrastructure fb_content_type-article" id="post-24128">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/10-Years-Meta-Python-HERO-large.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-06-30T09:00:46-07:00">
|
||
JUN 30, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/06/30/open-source/10-years-of-metas-commitment-to-python/" rel="bookmark" aria-label="Read More about 10 Years of Meta’s Commitment to Python">
|
||
10 Years of Meta’s Commitment to Python </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
</div><!-- /.row -->
|
||
</div><!--/.article-grids -->
|
||
|
||
</div><!-- /.related-articles -->
|
||
|
||
|
||
<!-- include the carousel nav here. -->
|
||
|
||
|
||
|
||
|
||
</main><!-- #main -->
|
||
|
||
</div><!-- #primary -->
|
||
|
||
<!-- Do the right sidebar check. -->
|
||
|
||
|
||
|
||
|
||
<div class="col-md-4 widget-area" id="right-sidebar" role="complementary">
|
||
<aside id="fb_related_post_widget-4" class="widget widget_fb_related_post_widget"><h3 class="widget-title">Related Posts</h3><hr>
|
||
<div class="aside-related-projects"><div class="article-grid-redesign">
|
||
<article class="post-22498 post type-post status-publish has-post-thumbnail hentry" id="post-22498">
|
||
<a href="https://engineering.fb.com/2025/05/20/web/metas-full-stack-hhvm-optimizations-for-genai/" rel="bookmark" aria-label="Read More about Meta's Full-stack HHVM optimizations for GenAI">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2023/10/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-201-Teale.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2023/10/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-201-Teale.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2023/10/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-201-Teale.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="May 20, 2025">May 20, 2025</time></span>
|
||
<h4 class="entry-title">Meta's Full-stack HHVM optimizations for GenAI</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div><div class="article-grid-redesign">
|
||
<article class="post-21330 post type-post status-publish has-post-thumbnail hentry" id="post-21330">
|
||
<a href="https://engineering.fb.com/2024/06/12/data-infrastructure/training-large-language-models-at-scale-meta/" rel="bookmark" aria-label="Read More about How Meta trains large language models at scale">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2024/06/Training-LLMs-at-Scale-Hero.png?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2024/06/Training-LLMs-at-Scale-Hero.png?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2024/06/Training-LLMs-at-Scale-Hero.png?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Jun 12, 2024">Jun 12, 2024</time></span>
|
||
<h4 class="entry-title">How Meta trains large language models at scale</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div><div class="article-grid-redesign">
|
||
<article class="post-21376 post type-post status-publish has-post-thumbnail hentry" id="post-21376">
|
||
<a href="https://engineering.fb.com/2024/06/19/data-infrastructure/parameter-vulnerability-factor-pvf-ai-silent-data-corruption/" rel="bookmark" aria-label="Read More about PVF: A novel metric for understanding AI systems’ vulnerability against SDCs in model parameters">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2023/10/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-201-Teale.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2023/10/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-201-Teale.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2023/10/Eng-Blog-Self-Serve-Hero-Images-DEBUGGING-201-Teale.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Jun 19, 2024">Jun 19, 2024</time></span>
|
||
<h4 class="entry-title">PVF: A novel metric for understanding AI systems’ vulnerability against SDCs in model parameters</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div></div></aside><aside id="wpb_widget-3" class="widget widget_wpb_widget"><h3 class="widget-title">Related Positions</h3><hr><ul class="available-positions-list"> </ul>
|
||
<a href="https://www.metacareers.com">
|
||
<div class="btn available-positions-btn">
|
||
See All Jobs </div>
|
||
</a>
|
||
</aside>
|
||
</div><!-- #secondary -->
|
||
|
||
|
||
|
||
<!-- <php get_template_part('content-single-side-bar'); ?> -->
|
||
|
||
<!-- Do the right sidebar check. -->
|
||
|
||
</div><!-- Container end -->
|
||
|
||
</div><!-- Wrapper end -->
|
||
|
||
</div>
|
||
|
||
<!-- ******************* The Footer Full-width Widget Area ******************* -->
|
||
|
||
<div class="wrapper wrapper-community-outer">
|
||
<div class="wrapper-community">
|
||
|
||
<div class="community" tabindex="-1">
|
||
|
||
<div class="row">
|
||
<!-- community section is generated through WordPress widget -->
|
||
<div id="wpb_widget-2" class="footer-widget widget_wpb_widget widget-count-3 col-md-4"><h3 class="widget-title">Available Positions</h3><hr><ul class="available-positions-list"> </ul>
|
||
<a href="https://www.metacareers.com">
|
||
<div class="btn available-positions-btn">
|
||
See All Jobs </div>
|
||
</a>
|
||
</div><!-- .footer-widget --><div id="custom_html-2" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Technology at Meta</h3><div class="textwidget custom-html-widget"><ul class="stay-connected-lists">
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-fb-engineering" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Engineering at Meta - X
|
||
</div>
|
||
<a onclick='return twitterFollow("Meta_Engineers")'>
|
||
<button class="btn stay-connected-btn">
|
||
Follow
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-AI" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
AI at Meta
|
||
</div>
|
||
<a href="https://ai.meta.com/blog/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta Quest Blog
|
||
</div>
|
||
<a href="https://www.meta.com/blog/quest/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta for Developers
|
||
</div>
|
||
<a href="https://developers.facebook.com/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-bug-bounty" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta Bug Bounty
|
||
</div>
|
||
<a href="https://bugbounty.meta.com/">
|
||
<button class="btn stay-connected-btn">
|
||
Learn more
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
|
||
<li class="stay-connected-item">
|
||
<img class="stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/rss.png" alt="footer-rss" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
RSS
|
||
</div>
|
||
<a href="https://code.facebook.com/posts/rss/">
|
||
<button class="btn stay-connected-btn">
|
||
Subscribe
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div></div><!-- .footer-widget --><div id="custom_html-3" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Open Source</h3><div class="textwidget custom-html-widget"><p class="opensource-desc">
|
||
Meta believes in building community through open source technology. Explore our latest projects in Artificial Intelligence, Data Infrastructure, Development Tools, Front End, Languages, Platforms, Security, Virtual Reality, and more.
|
||
</p>
|
||
<ul class="opensource-cats">
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/android.png" alt="android" loading="lazy" width="50" height="59">
|
||
<div class="opensource-cap">
|
||
ANDROID
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/ios.png" alt="ios" loading="lazy" width="48" height="58">
|
||
<div class="opensource-cap">
|
||
iOS
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/web.png" alt="web" loading="lazy" width="55" height="49">
|
||
<div class="opensource-cap">
|
||
WEB
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/backend.png" alt="backend" loading="lazy" width="56" height="51">
|
||
<div class="opensource-cap">
|
||
BACKEND
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/hardware.png" alt="hardware" loading="lazy" width="55" height="44">
|
||
<div class="opensource-cap">
|
||
HARDWARE
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
|
||
<button class="btn opensource-btn" onclick="window.location.href='https://opensource.fb.com/'">
|
||
Learn More
|
||
</button>
|
||
|
||
</div></div><!-- .footer-widget -->
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</div><!-- #wrapper-footer-full -->
|
||
|
||
|
||
<div class="wrapper-footer" id="wrapper-footer">
|
||
<div class="inner-container">
|
||
<div class="footer">
|
||
<div class="row">
|
||
<div class="col-md-12">
|
||
<footer class="meta-footer" id="meta-footer">
|
||
<div class="site-info row justify-content-between">
|
||
<div class="col-md-5 meta-logo-container">
|
||
<a href="https://about.facebook.com/">
|
||
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/meta_logo_full.svg" width="89" height="18" alt="Meta" />
|
||
</a>
|
||
<div id="text-2" class="widget widget_text"> <div class="textwidget"><p>Engineering at Meta is a technical news resource for engineers interested in how we solve large-scale technical challenges at Meta.</p>
|
||
</div>
|
||
</div> </div>
|
||
<div class="col-md-6 footer-meta-links-container">
|
||
<ul class="meta-footer-links">
|
||
<li class="footer-link"><a href="https://engineering.fb.com">Home</a></li><li class="footer-link"><a href="https://about.meta.com/">Company Info</a></li><li class="footer-link"><a href="https://www.metacareers.com/?ref=engineering.fb.com">Careers</a></li> </ul>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="footer">
|
||
<div class="row">
|
||
|
||
<div class="col-md-12">
|
||
|
||
<footer class="site-footer" id="colophon">
|
||
<div class="inner-container">
|
||
<div class="site-info row justify-content-between">
|
||
|
||
<div class="col-md-4 copyright-container">
|
||
<div class="copyright"> © 2026 Meta</div>
|
||
</div>
|
||
|
||
<!-- static footer -->
|
||
<div class="col-md-7 footer-links-container">
|
||
<ul class="footer-links">
|
||
<li class="footer-link"><a href="https://www.facebook.com/policies">Terms</a></li><li class="footer-link"><a href="https://www.facebook.com/privacy/policy">Privacy</a></li><li class="footer-link"><a href="/privacy">Cookies</a></li><li class="footer-link"><a href="https://www.facebook.com/help">Help</a></li> </ul>
|
||
</div>
|
||
</div>
|
||
</div><!-- .site-info -->
|
||
|
||
</footer><!-- #colophon -->
|
||
|
||
</div><!--col end -->
|
||
|
||
</div><!-- row end -->
|
||
|
||
</div><!-- //.inner-container -->
|
||
</div><!-- wrapper end -->
|
||
|
||
</div><!-- #page we need this extra closing tag here -->
|
||
|
||
<script type="speculationrules">
|
||
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/code-fb-com/*","/wp-content/themes/understrap/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
|
||
</script>
|
||
<div class="gdprconsent-container" id="GDPRConsentBar">
|
||
<div class="gdprconsent-wrapper">
|
||
<div class="gdprconsent-content">
|
||
<p>To help personalize content, tailor and measure ads and provide a safer experience, we use cookies. By clicking or navigating the site, you agree to allow our collection of information on and off Facebook through cookies. Learn more, including about available controls: <a href="/privacy">Cookie Policy</a></p>
|
||
<div class="gdprconsent-button">
|
||
<button type="button" data-gdpr-accept>Accept</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Twitter universal website tag code -->
|
||
<script>
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
|
||
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
|
||
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
|
||
// Insert Twitter Pixel ID and Standard Event data below
|
||
twq('init','o31u0');
|
||
twq('track','PageView');
|
||
});
|
||
}
|
||
</script>
|
||
<!-- End Twitter universal website tag code -->
|
||
|
||
|
||
<script type="text/javascript">
|
||
window.WPCOM_sharing_counts = {"https://engineering.fb.com/2025/07/22/data-infrastructure/how-meta-keeps-its-ai-hardware-reliable/":22663};
|
||
</script>
|
||
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KZRLGZ8" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
<noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=660742068067366&ev=PageView&noscript=1"/></noscript>
|
||
<script id="child-understrap-scripts-js-extra">
|
||
var loadmore_params = {"restfulURL":"https://engineering.fb.com/wp-json/fb/v1/loadmore","searchText":"","posts":"{\"page\":0,\"year\":2025,\"monthnum\":7,\"day\":22,\"name\":\"how-meta-keeps-its-ai-hardware-reliable\",\"category_name\":\"data-infrastructure\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","post_type":"post"};
|
||
var search_params = {"ajaxurl":"https://engineering.fb.com/wp-admin/admin-ajax.php","posts":"{\"page\":0,\"year\":2025,\"monthnum\":7,\"day\":22,\"name\":\"how-meta-keeps-its-ai-hardware-reliable\",\"category_name\":\"data-infrastructure\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","article_filter":"2f23ad5ce3"};
|
||
var more_articles = {"next_article":{"label":"Next","permalink":"https://engineering.fb.com/2025/07/23/security/policy-zones-meta-purpose-limitation-batch-processing-systems/","title":"Policy Zones: How Meta enforces purpose limitation at scale in batch processing systems","thumbnail":["https://engineering.fb.com/wp-content/uploads/2025/07/Policy-Zones-How-Meta-enforces-purpose-limitation-at-scale-in-batch-processing-systems.jpg?w=580&h=326&crop=1",580,326,true]},"prev_article":{"label":"Prev","permalink":"https://engineering.fb.com/2025/07/16/data-center-engineering/ai-make-lower-carbon-faster-curing-concrete/","title":"Using AI to make lower-carbon, faster-curing concrete","thumbnail":["https://engineering.fb.com/wp-content/uploads/2025/07/Decarbonizing-concrete-using-Metas-Bayesian-Optimization-AI-Eng-at-Meta.png?w=580&h=326&crop=1",580,326,true]}};
|
||
//# sourceURL=child-understrap-scripts-js-extra
|
||
</script>
|
||
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJx9zOEKwjAMBOAXsouDsekP8V1sMpraNmWpim9vNhQEwb933x08qvNSGpUGLVAmBS9Ibr5YnAFZG0TLAid0G+gyly7qDv4vbVQX1vyhXHy6oSErttc7FZRlraqk58wpfT//8CBy1bc451M/HaZxPxz7Ib4Aa4BGPQ==" ></script><script id="wp-i18n-js" src="https://engineering.fb.com/wp-includes/js/dist/i18n.min.js?ver=1dfe7db3940c23ea9216"></script>
|
||
<script id="wp-i18n-js-after">
|
||
wp.i18n.setLocaleData( { 'text direction\u0004ltr': [ 'ltr' ] } );
|
||
//# sourceURL=wp-i18n-js-after
|
||
</script>
|
||
<script id="jetpack-responsive-videos-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/jetpack_vendor/automattic/jetpack-classic-theme-helper/dist/responsive-videos/responsive-videos.js?minify=false&ver=1a16f71f21c6d197f51b"></script>
|
||
<script id="sharing-js-js-extra">
|
||
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":""};
|
||
//# sourceURL=sharing-js-js-extra
|
||
</script>
|
||
<script id="sharing-js-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/_inc/build/sharedaddy/sharing.min.js?ver=16.1.3"></script>
|
||
<script id="sharing-js-js-after">
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-facebook' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-threads' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-threads' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomthreads', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-linkedin' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-x' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-x' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomx', 'menubar=1,resizable=1,width=600,height=350' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-bluesky' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-bluesky' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcombluesky', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-mastodon' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-mastodon' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcommastodon', 'menubar=1,resizable=1,width=460,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
//# sourceURL=sharing-js-js-after
|
||
</script>
|
||
<script id="video-analytics-js-after">
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
let youtube = document.querySelectorAll('[src*="youtube-nocookie"]');
|
||
for (let i = 0; i < youtube.length; i++) {
|
||
youtube[i].src = youtube[i].src.replace('youtube-nocookie.com', 'youtube.com');
|
||
}
|
||
let vimeo = document.querySelectorAll('[src*="vimeo"]');
|
||
for (let i = 0; i < vimeo.length; i++) {
|
||
vimeo[i].src = vimeo[i].src.replace('&dnt=true', '');
|
||
}
|
||
});
|
||
}
|
||
//# sourceURL=video-analytics-js-after
|
||
</script>
|
||
<script id="wp-emoji-settings" type="application/json">
|
||
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://engineering.fb.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
|
||
</script>
|
||
<script type="module">
|
||
/*! This file is auto-generated */
|
||
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
|
||
//# sourceURL=https://engineering.fb.com/wp-includes/js/wp-emoji-loader.min.js
|
||
</script>
|
||
|
||
</body>
|
||
|
||
</html>
|