1289 lines
108 KiB
HTML
1289 lines
108 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en-US">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||
<meta name="mobile-web-app-capable" content="yes">
|
||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||
<meta name="apple-mobile-web-app-title" content="Engineering at Meta - Engineering at Meta Blog">
|
||
<meta property="fb:app_id" content="1425766027653270" />
|
||
<link rel="profile" href="http://gmpg.org/xfn/11">
|
||
<link rel="pingback" href="https://engineering.fb.com/xmlrpc.php">
|
||
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
|
||
<script async src="https://engineering.fb.com/wp-content/themes/code-fb-com/dist/js/optimize.min.js"></script>
|
||
|
||
<!-- This site is optimized with the Yoast SEO Premium plugin v19.3 (Yoast SEO v27.3) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
|
||
<title>How Facebook deals with PCIe faults to keep our data centers running reliably - Engineering at Meta</title>
|
||
<meta name="description" content="Facebook has developed a new solution to detect, diagnose, remediate, and repair faults in PCIe-based hardware components." />
|
||
<link rel="canonical" href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/" />
|
||
<meta name="twitter:label1" content="Written by" />
|
||
<meta name="twitter:data1" content="Ashwin Poojary, Bill Holland, Makan Diarra, Ray Park" />
|
||
<meta name="twitter:label2" content="Est. reading time" />
|
||
<meta name="twitter:data2" content="9 minutes" />
|
||
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/#article","isPartOf":{"@id":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/"},"author":[{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/e190664611be84065041b94c6b952107"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/68607b259ff1f37effa359f6b6c9e976"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/c700ef1dc548aba9397c4c086c7f275a"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/0795946138649b50d91db4fc309187df"}],"headline":"How Facebook deals with PCIe faults to keep our data centers running reliably","datePublished":"2021-06-02T16:00:56+00:00","mainEntityOfPage":{"@id":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/"},"wordCount":2002,"publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"image":{"@id":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/05\/Facebook010_data-center.jpg","articleSection":["Data Center Engineering","Production Engineering"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/","url":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/","name":"How Facebook deals with PCIe faults to keep our data centers running reliably - Engineering at Meta","isPartOf":{"@id":"https:\/\/engineering.fb.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/#primaryimage"},"image":{"@id":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/05\/Facebook010_data-center.jpg","datePublished":"2021-06-02T16:00:56+00:00","description":"Facebook has developed a new solution to detect, diagnose, remediate, and repair faults in PCIe-based hardware components.","breadcrumb":{"@id":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/#primaryimage","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/05\/Facebook010_data-center.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/05\/Facebook010_data-center.jpg","width":5472,"height":3648},{"@type":"BreadcrumbList","@id":"https:\/\/engineering.fb.com\/2021\/06\/02\/data-center-engineering\/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/engineering.fb.com\/"},{"@type":"ListItem","position":2,"name":"How Facebook deals with PCIe faults to keep our data centers running reliably"}]},{"@type":"WebSite","@id":"https:\/\/engineering.fb.com\/#website","url":"https:\/\/engineering.fb.com\/","name":"Engineering at Meta","description":"Engineering at Meta Blog","publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/engineering.fb.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/engineering.fb.com\/#organization","name":"Meta","url":"https:\/\/engineering.fb.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","width":29011,"height":12501,"caption":"Meta"},"image":{"@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/Engineering\/","https:\/\/x.com\/fb_engineering"]},[],{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/e190664611be84065041b94c6b952107","name":"Ashwin Poojary","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Ashwin Poojary"},"url":"https:\/\/engineering.fb.com\/author\/ashwin-poojary\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/68607b259ff1f37effa359f6b6c9e976","name":"Bill Holland","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Bill Holland"},"url":"https:\/\/engineering.fb.com\/author\/bill-holland\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/c700ef1dc548aba9397c4c086c7f275a","name":"Makan Diarra","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Makan Diarra"},"url":"https:\/\/engineering.fb.com\/author\/makan-diarra\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/0795946138649b50d91db4fc309187df","name":"Ray Park","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Ray Park"},"url":"https:\/\/engineering.fb.com\/author\/ray-park\/"}]}</script>
|
||
<!-- / Yoast SEO Premium plugin. -->
|
||
|
||
|
||
<link rel='dns-prefetch' href='//secure.gravatar.com' />
|
||
<link rel='dns-prefetch' href='//v0.wordpress.com' />
|
||
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta » Feed" href="https://engineering.fb.com/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta » Comments Feed" href="https://engineering.fb.com/comments/feed/" />
|
||
<script async src="https://www.googleoptimize.com/optimize.js?id=OPT-TPNDQC5" onerror="dataLayer.hide.end && dataLayer.hide.end()"></script>
|
||
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2021%2F06%2F02%2Fdata-center-engineering%2Fhow-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably%2F" />
|
||
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2021%2F06%2F02%2Fdata-center-engineering%2Fhow-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably%2F&format=xml" />
|
||
<style id="wp-img-auto-sizes-contain-inline-css">
|
||
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
|
||
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
|
||
</style>
|
||
<style id="wp-emoji-styles-inline-css">
|
||
|
||
img.wp-smiley, img.emoji {
|
||
display: inline !important;
|
||
border: none !important;
|
||
box-shadow: none !important;
|
||
height: 1em !important;
|
||
width: 1em !important;
|
||
margin: 0 0.07em !important;
|
||
vertical-align: -0.1em !important;
|
||
background: none !important;
|
||
padding: 0 !important;
|
||
}
|
||
/*# sourceURL=wp-emoji-styles-inline-css */
|
||
</style>
|
||
<style id="wp-block-library-inline-css">
|
||
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
|
||
|
||
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
|
||
</style>
|
||
<style id="classic-theme-styles-inline-css">
|
||
/*! This file is auto-generated */
|
||
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
|
||
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
|
||
</style>
|
||
|
||
<style id="global-styles-inline-css">
|
||
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
|
||
/*# sourceURL=global-styles-inline-css */
|
||
</style>
|
||
|
||
<link rel='stylesheet' id='all-css-12' href='https://engineering.fb.com/_static/??-eJyNjUsOwjAMRC9EaiqghQXiKFVqhzSQn+pEiNuTFJBAbNjZTzNv4BYFBp+UTxBt1sYzFKQpzpVz5WQ4QSUv0CDzCj6KaVJOMWAgJc5jwe7ZKTnAyVgSS6Jxxv90XRbv3YtKUeJVtF3TgguUbZHyJGdFkui+nMbrvxWD8QhjLvvAAY20wgYd+OupspM7tv3+sO533Wb7AImQY4U=' type='text/css' media='all' />
|
||
<script id="gdprconsent-js" src="https://engineering.fb.com/wp-content/plugins/wp-gdpr-consent/dist/gdprconsent.js?ver=2.0.0"></script>
|
||
<script id="gdprconsent-js-after">
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack('https://www.googletagmanager.com/gtag/js?id=391063774');
|
||
window.gdprSafeTrack(function() {
|
||
var google_analytics_g4 = decodeURIComponent( '391063774' );
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
gtag('js', new Date());
|
||
gtag('config', google_analytics_g4);
|
||
});
|
||
}
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
var google_tag_manager = decodeURIComponent( 'GTM-KZRLGZ8' );
|
||
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||
})(window,document,'script','dataLayer',google_tag_manager);
|
||
});
|
||
}
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
var facebook_pixel = decodeURIComponent( '660742068067366' );
|
||
!function(f,b,e,v,n,t,s)
|
||
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
|
||
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
|
||
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
|
||
n.queue=[];t=b.createElement(e);t.async=!0;
|
||
t.src=v;s=b.getElementsByTagName(e)[0];
|
||
s.parentNode.insertBefore(t,s)}(window, document,'script',
|
||
'https://connect.facebook.net/en_US/fbevents.js');
|
||
fbq('init', facebook_pixel);
|
||
fbq('track', 'PageView');
|
||
});
|
||
}
|
||
//# sourceURL=gdprconsent-js-after
|
||
</script>
|
||
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJzTLy/QzcxLzilNSS3WzwKiwtLUokoopZebmaeXVayjj0+Rbm5melFiSSpUsX2uraG5hbmZgYmloUkWAK+3Iic=" ></script><script type="text/javascript" src="https://engineering.fb.com/wp-content/themes/understrap/js/popper.min.js?m=1773259670g" ></script><link rel="https://api.w.org/" href="https://engineering.fb.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.fb.com/wp-json/wp/v2/posts/17580" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.fb.com/xmlrpc.php?rsd" />
|
||
<meta name="generator" content="WordPress 7.1" />
|
||
<link rel='shortlink' href='https://wp.me/pa0Lhq-4zy' />
|
||
<link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/Optimistic_Display_W_Md.woff2" as="font" type="font/woff2" crossorigin="anonymous">
|
||
<!-- Jetpack Open Graph Tags -->
|
||
<meta property="og:type" content="article" />
|
||
<meta property="og:title" content="How Facebook deals with PCIe faults to keep our data centers running reliably" />
|
||
<meta property="og:url" content="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/" />
|
||
<meta property="og:description" content="Peripheral component interconnect express (PCIe) hardware continues to push the boundaries of computing thanks to advances in transfer speeds, the number of available lanes for simultaneous data de…" />
|
||
<meta property="article:published_time" content="2021-06-02T16:00:56+00:00" />
|
||
<meta property="article:modified_time" content="2021-06-01T16:08:18+00:00" />
|
||
<meta property="og:site_name" content="Engineering at Meta" />
|
||
<meta property="og:image" content="https://engineering.fb.com/wp-content/uploads/2021/05/Facebook010_data-center.jpg" />
|
||
<meta property="og:image:width" content="5472" />
|
||
<meta property="og:image:height" content="3648" />
|
||
<meta property="og:image:alt" content="" />
|
||
<meta property="og:locale" content="en_US" />
|
||
<meta name="twitter:text:title" content="How Facebook deals with PCIe faults to keep our data centers running reliably" />
|
||
<meta name="twitter:card" content="summary" />
|
||
|
||
<!-- End Jetpack Open Graph Tags -->
|
||
<style id="wp-custom-css">
|
||
.table-of-contents {
|
||
position: sticky;
|
||
top: 20px;
|
||
background: #fff;
|
||
padding: 1em;
|
||
border: 1px solid #ddd;
|
||
border-radius: 8px;
|
||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||
max-width: 250px;
|
||
}
|
||
</style>
|
||
|
||
<link rel="shortcut icon" href="https://engineering.fb.com/wp-content/themes/code-fb-com/favicon.ico" />
|
||
</head>
|
||
|
||
<body class="wp-singular post-template-default single single-post postid-17580 single-format-standard wp-theme-understrap wp-child-theme-code-fb-com group-blog">
|
||
|
||
<div class="hfeed site" id="page">
|
||
|
||
<!-- ******************* The Navbar Area ******************* -->
|
||
<div class="wrapper-fluid wrapper-navbar" id="wrapper-navbar">
|
||
<a class="skip-link screen-reader-text sr-only" href="#content">
|
||
Skip to content </a>
|
||
|
||
<nav class="navbar navbar-expand-std">
|
||
|
||
<div class="nav-container">
|
||
|
||
|
||
<!-- Your site title as branding in the menu -->
|
||
|
||
<div class="nav-top-row">
|
||
|
||
<h1 class="navbar-brand mb-0">
|
||
<a rel="home" href="https://engineering.fb.com/" title="Engineering at Meta">
|
||
<img class="navbar-brand-logo" src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/logo-meta.svg" class="navbar-brand-logo" alt="Engineering at Meta" width="215" height="40" loading="lazy">
|
||
</a>
|
||
</h1>
|
||
|
||
<!-- end custom logo -->
|
||
<label for="search" class="screen-reader-text">Search this site</label>
|
||
<input type="text" id="search" name="search" class="form-control-lg navbar-search" autocomplete="off">
|
||
|
||
<button class="navbar-toggler navbar-searchbar-toggler"
|
||
type="button"
|
||
data-toggle="collapse"
|
||
data-target="#searchbarDropdown"
|
||
aria-controls="searchbarDropdown"
|
||
aria-expanded="false"
|
||
aria-label="Toggle navigation">
|
||
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/icon-search.svg" class="navbar-mobile-search-icon" loading="lazy" width="24" height="25" />
|
||
</button>
|
||
|
||
<!-- desktop language menu (mobile language menu is implemented via jQuery)-->
|
||
<div class="notranslate nice-select navbar-language form-control">
|
||
</div>
|
||
|
||
<button class="navbar-toggler collapsed navbar-menu-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
|
||
</div>
|
||
|
||
<div class="navbar-border"></div>
|
||
|
||
<!-- The WordPress Menu goes here -->
|
||
<div id="navbarNavDropdown" class="collapse navbar-collapse"><ul id="main-menu" class="navbar-nav"><li id="menu-item-60" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-60 dropdown"><a title="Open Source" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Open Source <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-12330" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-12330"><a title="Open Source" href="https://engineering.fb.com/category/open-source/" class="nav-link">Open Source</a></li>
|
||
<li id="menu-item-12331" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-12331"><a title="Meta Open Source" href="https://opensource.fb.com" class="nav-link">Meta Open Source</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2686" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2686 dropdown"><a title="Platforms" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Platforms <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-65" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-65"><a title="Android" href="https://engineering.fb.com/category/android/" class="nav-link">Android</a></li>
|
||
<li id="menu-item-66" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-66"><a title="iOS" href="https://engineering.fb.com/category/ios/" class="nav-link">iOS</a></li>
|
||
<li id="menu-item-67" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-67"><a title="Web" href="https://engineering.fb.com/category/web/" class="nav-link">Web</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2682" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2682 dropdown"><a title="Infrastructure Systems" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Infrastructure Systems <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-218" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-218"><a title="Core Infra" href="https://engineering.fb.com/category/core-infra/" class="nav-link">Core Infra</a></li>
|
||
<li id="menu-item-219" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-219"><a title="Data Infrastructure" href="https://engineering.fb.com/category/data-infrastructure/" class="nav-link">Data Infrastructure</a></li>
|
||
<li id="menu-item-220" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-220"><a title="DevInfra" href="https://engineering.fb.com/category/developer-tools/" class="nav-link">DevInfra</a></li>
|
||
<li id="menu-item-226" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-226"><a title="Production Engineering" href="https://engineering.fb.com/category/production-engineering/" class="nav-link">Production Engineering</a></li>
|
||
<li id="menu-item-227" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-227"><a title="Security & Privacy" href="https://engineering.fb.com/category/security/" class="nav-link">Security & Privacy</a></li>
|
||
<li id="menu-item-18607" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18607"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/systems-infrastructure/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2683" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2683 dropdown"><a title="Physical Infrastructure" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Physical Infrastructure <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-215" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-215"><a title="Connectivity" href="https://engineering.fb.com/category/connectivity/" class="nav-link">Connectivity</a></li>
|
||
<li id="menu-item-216" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-216"><a title="Data Center Engineering" href="https://engineering.fb.com/category/data-center-engineering/" class="nav-link">Data Center Engineering</a></li>
|
||
<li id="menu-item-217" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-217"><a title="Networking & Traffic" href="https://engineering.fb.com/category/networking-traffic/" class="nav-link">Networking & Traffic</a></li>
|
||
<li id="menu-item-18608" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18608"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/networking-connectivity/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2684" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2684 dropdown"><a title="Video Engineering & AR/VR" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Video Engineering & AR/VR <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-225" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-225"><a title="Video Engineering" href="https://engineering.fb.com/category/video-engineering/" class="nav-link">Video Engineering</a></li>
|
||
<li id="menu-item-223" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-223"><a title="Virtual Reality" href="https://engineering.fb.com/category/virtual-reality/" class="nav-link">Virtual Reality</a></li>
|
||
<li id="menu-item-13388" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13388"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/augmented-reality-virtual-reality/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2685" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2685 dropdown"><a title="Artificial Intelligence" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Artificial Intelligence <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-222" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-222"><a title="ML Applications" href="https://engineering.fb.com/category/ml-applications/" class="nav-link">ML Applications</a></li>
|
||
<li id="menu-item-221" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-221"><a title="AI Research" href="https://engineering.fb.com/category/ai-research/" class="nav-link">AI Research</a></li>
|
||
<li id="menu-item-13387" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13387"><a title="Research Publications" href="https://ai.facebook.com/results/?content_types%5B0%5D=publication" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-228" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-228"><a title="Watch Videos" href="/videos" class="nav-link">Watch Videos</a></li>
|
||
</ul></div> </div><!-- .container -->
|
||
|
||
<div class="search-container collapse" id="searchbarDropdown">
|
||
<div class="searchbar-search">
|
||
<div class="searchbar-container">
|
||
|
||
<div class="navbar-mobile-search-margin"></div>
|
||
<input type="text" name="search" placeholder="Search" class="form-control-lg navbar-mobile-search" autofocus autocomplete="off">
|
||
<button class="searchbar-clear">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</nav><!-- .site-navigation -->
|
||
|
||
</div><!-- .wrapper-navbar end -->
|
||
|
||
<div class="wrapper" id="single-wrapper">
|
||
|
||
<div class="single-container row" id="content" tabindex="-1">
|
||
|
||
<!-- Do the left sidebar check. -->
|
||
|
||
|
||
|
||
<div class="content-area" id="primary">
|
||
<main class="site-main" id="main">
|
||
|
||
|
||
|
||
<article class="post-17580 post type-post status-publish format-standard has-post-thumbnail hentry category-data-center-engineering category-production-engineering fb_content_type-article" id="post-17580">
|
||
|
||
<header class="entry-header">
|
||
<div class="entry-meta">
|
||
POSTED ON <span class="posted-on entry-date"><time class="published updated" datetime="2021-06-02">JUNE 2, 2021</time></span> TO <span class="cat-links"><a href="https://engineering.fb.com/category/data-center-engineering/" rel="category tag">Data Center Engineering</a>, <a href="https://engineering.fb.com/category/production-engineering/" rel="category tag">Production Engineering</a></span>
|
||
<h1 class="entry-title"><div>How Facebook deals with PCIe faults to keep our data centers running reliably</div></h1> </div><!-- .entry-meta -->
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
<figure id="post-feat-image-container">
|
||
<img width="5472" height="3648" src="https://engineering.fb.com/wp-content/uploads/2021/05/Facebook010_data-center.jpg" class="wp-post-image" alt="" title="Facebook010_data center" /> </figure>
|
||
|
||
<div class="entry-authors-social">
|
||
<div class="entry-authors">
|
||
By <a href="https://engineering.fb.com/author/ashwin-poojary/" title="Posts by Ashwin Poojary" class="author url fn" rel="author">Ashwin Poojary</a>, <a href="https://engineering.fb.com/author/bill-holland/" title="Posts by Bill Holland" class="author url fn" rel="author">Bill Holland</a>, <a href="https://engineering.fb.com/author/makan-diarra/" title="Posts by Makan Diarra" class="author url fn" rel="author">Makan Diarra</a>, <a href="https://engineering.fb.com/author/ray-park/" title="Posts by Ray Park" class="author url fn" rel="author">Ray Park</a> </div>
|
||
</div>
|
||
|
||
|
||
<div class="entry-content">
|
||
|
||
<p><span style="font-weight: 400;">Peripheral component interconnect express (PCIe) hardware continues to push the boundaries of computing thanks to advances in transfer speeds, the number of available lanes for simultaneous data delivery, and a comparatively small footprint on motherboards. Today, PCIe connectivity-based hardware delivers faster data transfers and is one of the de facto methods to connect components to servers.</span></p>
|
||
<p><span style="font-weight: 400;">Our data centers contain millions of PCIe-based hardware components — including ASIC-based accelerators for video and inference, GPUs, NICs, and SSDs — connected either directly into a PCI slot on a server’s motherboard or through a PCIe switch like a carrier card.</span></p>
|
||
<p><span style="font-weight: 400;">As with any hardware, PCIe-based components are susceptible to different types of hardware-, firmware-, or software-related failures and performance degradation. The variety of components and vendors, array of failures, and the challenges of scale make monitoring, collecting data, and performing fault isolation for PCIe-based components challenging.</span></p>
|
||
<p><span style="font-weight: 400;">We’ve developed a solution to detect, diagnose, remediate, and repair these issues. Since we’ve implemented it, this methodology has helped make our hardware fleet more reliable, resilient, and performant. And we believe the wider industry can benefit from the same information, strategies, and help build industry standards around this common problem.</span></p>
|
||
<figure id="attachment_17581" aria-describedby="caption-attachment-17581" style="width: 1024px" class="wp-caption alignnone"><img decoding="async" class="size-large wp-image-17581" src="https://engineering.fb.com/wp-content/uploads/2021/05/Tioga-Pass-Type-3_00_Full-Body-Cover-Background_IMG_5996.jpg?w=1024" alt="" width="1024" height="682" srcset="https://engineering.fb.com/wp-content/uploads/2021/05/Tioga-Pass-Type-3_00_Full-Body-Cover-Background_IMG_5996.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2021/05/Tioga-Pass-Type-3_00_Full-Body-Cover-Background_IMG_5996.jpg?resize=916,611 916w, https://engineering.fb.com/wp-content/uploads/2021/05/Tioga-Pass-Type-3_00_Full-Body-Cover-Background_IMG_5996.jpg?resize=768,512 768w, https://engineering.fb.com/wp-content/uploads/2021/05/Tioga-Pass-Type-3_00_Full-Body-Cover-Background_IMG_5996.jpg?resize=1024,682 1024w, https://engineering.fb.com/wp-content/uploads/2021/05/Tioga-Pass-Type-3_00_Full-Body-Cover-Background_IMG_5996.jpg?resize=1536,1024 1536w, https://engineering.fb.com/wp-content/uploads/2021/05/Tioga-Pass-Type-3_00_Full-Body-Cover-Background_IMG_5996.jpg?resize=96,64 96w, https://engineering.fb.com/wp-content/uploads/2021/05/Tioga-Pass-Type-3_00_Full-Body-Cover-Background_IMG_5996.jpg?resize=192,128 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-17581" class="wp-caption-text">Facebook’s data centers employ a range of PCIe-based hardware components.</figcaption></figure>
|
||
<h2><span style="font-weight: 400;">Our tools for addressing PCIe faults</span></h2>
|
||
<p><span style="font-weight: 400;">First, let’s outline the tools we use:</span></p>
|
||
<ul>
|
||
<li style="font-weight: 400;" aria-level="1"><a href="https://engineering.fb.com/2020/08/05/open-source/pcicrawler/"><b>PCIcrawler</b></a><b>:</b><span style="font-weight: 400;"> An open source, Python-based command line interface tool that can be used to display, filter, and export information about PCI or PCIe buses and devices, including PCI topology and PCIe Advanced Error Reporting (AER) errors. This tool produces visually appealing, treelike outputs for easy debugging as well as machine parsable json output that can be consumed by tools for deployment at scale.</span></li>
|
||
</ul>
|
||
<ul>
|
||
<li style="font-weight: 400;" aria-level="1"><a href="https://engineering.fb.com/2020/12/09/data-center-engineering/how-facebook-keeps-its-large-scale-infrastructure-hardware-up-and-running/"><b>MachineChecker</b></a><b>:</b><span style="font-weight: 400;"> An in-house tool for quickly evaluating the production worthiness of servers from a hardware standpoint. MachineChecker helps detect and diagnose hardware problems. It can be run as a command line input tool. It also lives as a library and a service.</span></li>
|
||
</ul>
|
||
<ul>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">An in-house tool for taking a snapshot of the target host’s hardware configuration along with hardware modeling.</span></li>
|
||
</ul>
|
||
<ul>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">An in-house utility service used to parse the custom </span><a href="https://man7.org/linux/man-pages/man1/dmesg.1.html"><span style="font-weight: 400;">dmesg</span></a><span style="font-weight: 400;"> and SELs to detect and report PCIe errors on millions of servers. This tool parses the logs on the server at regular intervals and records the rate of correctable errors on a file on the corresponding server. The rate is recorded per 10 minutes, per 30 minutes, per hour, per six hours, and per day. This rate is used to decide which servers have exceeded the configured tolerable PCIe-corrected error rate threshold depending on the platform and the service.</span></li>
|
||
</ul>
|
||
<ul>
|
||
<li style="font-weight: 400;" aria-level="1"><a href="https://github.com/ipmitool/"><b>IPMI Tool</b></a><b>:</b><span style="font-weight: 400;"> An open source utility for managing and configuring devices that support the Intelligent Platform Management Interface (IPMI). IPMI is an open standard for monitoring, logging, recovery, inventory, and control of hardware that is implemented independent of the main CPU, BIOS, and OS. It’s mainly used to manually extract </span><a href="https://www.intel.com/content/dam/support/us/en/documents/server-products/SEL_TroubleshootingGuide.pdf"><span style="font-weight: 400;">System Event Logs</span></a><span style="font-weight: 400;"> (SELs) for inspection, debugging, and study.</span></li>
|
||
</ul>
|
||
<ul>
|
||
<li style="font-weight: 400;" aria-level="1"><a href="https://github.com/openbmc/"><b>The OpenBMC Project</b></a><b>:</b> <span style="font-weight: 400;">A Linux distribution for embedded devices that have a baseboard management controller (BMC).</span></li>
|
||
</ul>
|
||
<ul>
|
||
<li style="font-weight: 400;" aria-level="1"><b>Facebook auto remediation (</b><a href="https://engineering.fb.com/2016/07/11/production-engineering/making-facebook-self-healing-automating-proactive-rack-maintenance/"><b>FBAR</b></a><b>):</b><span style="font-weight: 400;"> A system and a set of daemons that execute code automatically in response to detected software and hardware signals on individual servers. Every day, without human intervention, FBAR takes faulty servers out of production and sends requests to our data center teams to perform physical hardware repairs, making isolated failures a nonissue.</span></li>
|
||
</ul>
|
||
<ul>
|
||
<li style="font-weight: 400;" aria-level="1"><a href="https://research.fb.com/wp-content/uploads/2016/11/scuba-diving-into-data-at-facebook.pdf"><b>Scuba</b></a><b>:</b><span style="font-weight: 400;"> A fast, scalable, distributed, in-memory database built at Facebook. It is the data management system we use for most of our real-time analysis.</span></li>
|
||
</ul>
|
||
<h2><span style="font-weight: 400;">How we studied PCIe faults</span></h2>
|
||
<p><span style="font-weight: 400;">The sheer variety of PCIe hardware components (ASICs, NICs, SSDs, etc.) makes studying PCIe issues a daunting task. These components can have different vendors, firmware versions, and different applications running on them. On top of this, the applications themselves might have different compute and storage needs, usage profiles, and tolerances.</span></p>
|
||
<p><span style="font-weight: 400;">By leveraging the tools listed above, we’ve been conducting studies to ameliorate these challenges and ascertain the root cause of PCIe hardware failures and performance degradation.</span></p>
|
||
<p><span style="font-weight: 400;">Some of the issues were obvious. PCIe fatal uncorrected errors, for example, are definitely bad, even if there is only one instance on a particular server. MachineChecker can detect this and mark the faulty hardware (ultimately leading to it being replaced).</span></p>
|
||
<p><span style="font-weight: 400;">Depending on the error conditions, uncorrectable errors are further classified into nonfatal errors and fatal errors. Nonfatal errors are ones that cause a particular transaction to be unreliable, but the PCIe link itself is fully functional. Fatal errors, on the other hand, cause the link to be unreliable. Based on our experience, we’ve found that for any uncorrected PCIe error, swapping the hardware component (and sometimes the motherboard) is the most effective action.</span></p>
|
||
<p><span style="font-weight: 400;">Other issues can seem innocuous at first. PCIe-corrected errors, for example, are correctable by definition and are mostly corrected well in practice. Correctable errors are supposed to pose no impact on the functionality of the interface. However, the rate at which correctable errors occur matters. And if the rate is beyond a particular threshold, it leads to a degradation in performance that is not acceptable for certain applications.</span></p>
|
||
<p><span style="font-weight: 400;">We conducted an in-depth study to correlate the performance degradation and system stalls to PCIe-corrected error rates. Determining the threshold is another challenge, since different platforms and different applications have different profiles and needs. We rolled out the PCIe Error Logging Service, observed the failures in the Scuba tables, and correlated events, system stalls, and PCIe faults to determine the thresholds for each platform. We’ve found that swapping hardware is the most effective solution when PCIe-corrected error rates cross a particular threshold.</span></p>
|
||
<p><span style="font-weight: 400;">PCIe defines two error-reporting paradigms: The baseline capability and the AER capability. The baseline capability is required of all PCIe components and provides a minimum defined set of error reporting requirements. The AER capability is implemented with a PCIe AER extended capability structure and provides more robust error reporting. The PCIe AER driver provides the infrastructure to support PCIe AER capability and we leveraged PCIcrawler to take advantage of this.</span></p>
|
||
<p><span style="font-weight: 400;">We recommend that every vendor adopt the PCIe AER functionality and PCIcrawler rather than relying on custom vendor tools, which lack generality. Custom tools are hard to parse and even harder to maintain. Moreover, integrating new vendors, new kernel versions, or new types of hardware requires a lot of time and effort.</span></p>
|
||
<p><span style="font-weight: 400;">Bad (down-negotiated) link speed (usually running at 1/2 or 1/4 of the expected speed) and bad (down-negotiated) link width (running at 1/2, 1/4, or even 1/8 of the expected link width) were other concerning PCIe faults. These faults can be difficult to detect without some sort of automated tool because the hardware is working, just not as optimally as it could.</span></p>
|
||
<p><span style="font-weight: 400;">Based on our study at scale, we found that most of these faults could be corrected by reseating hardware components. This is why we try this first before marking the hardware as faulty.</span></p>
|
||
<p><span style="font-weight: 400;">Since a small minority of these faults can be fixed by a reboot, we also record historical repair actions. We have special rules to identify repeat offenders. For example, if the same hardware component on the same server fails a predefined number of times in a predetermined time interval, after a predefined number of reseats, we automatically mark it as faulty and swap it out. In cases where the component swap does not fix it, we will have to resort to a motherboard swap.</span></p>
|
||
<p><span style="font-weight: 400;">We also keep an eye on the repair trend to identify nontypical failure rates. For example, in one case, by using data from custom Scuba tables and their illustrative graphs and timelines, we root-caused a down-negotiation issue to a specific firmware release from a specific vendor. We then worked with the vendor to roll out new firmware that fixed the issue.</span></p>
|
||
<p><span style="font-weight: 400;">It’s also important to rate-limit remediations and repairs as a safety net to prevent bugs in the code from mass draining and unprovisioning, which can result in service outages if not handled properly.</span></p>
|
||
<p><span style="font-weight: 400;">Using this overall methodology, we’ve been able to add hardware health coverage and fix several thousand servers and server components. Every week, we’ve been able to detect, diagnose, remediate, and repair various PCIe faults on hundreds of servers.</span></p>
|
||
<h2><span style="font-weight: 400;">Our PCIe fault workflow</span></h2>
|
||
<p><span style="font-weight: 400;">Here’s a step-by-step breakdown of our process for identifying and fixing PCIe faults:</span></p>
|
||
<ol>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">MachineChecker runs periodically as a service on the millions of hardware servers and switches in our production fleet. Some of the checks include PCIe link speed, PCIe link width, as well as PCIe-uncorrected and PCIe-corrected error rate checks.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">For a particular PCIe endpoint, we find its parent called upstream using PCIcrawler’s PCIe topology information. We consider both ends of a PCIe link.</span><span style="font-weight: 400;"><br />
|
||
</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">We leverage PCIcrawler’s output, which in turn depends on the generic registers LnkSta, LnkSta2, LnkCtl, and LnkCtl2.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">We calculate expected speed as:</span><span style="font-weight: 400;"><br />
|
||
</span><span style="font-weight: 400;">expected_speed = min (upstream_target_speed, endpoint_capable_speed, upstream_capable_speed).</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">We calculate current_speed as:</span><span style="font-weight: 400;"><br />
|
||
</span><span style="font-weight: 400;">current_speed = min (endpoint_current_speed, upstream_current_speed).</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">current_speed must be equal to expected_speed.<br />
|
||
</span>In other words, we should have the current speed of either end be equal to the minimum of the capable speeds, upstream capable, downstream capable, and upstream target speed.</li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">For PCIe link width, we calculate expected_width as:</span><span style="font-weight: 400;"><br />
|
||
</span><span style="font-weight: 400;">expected_width = min(pcie_upstream_device capable_width, pcie_endpoint_device capable width).</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">If the expected_width is less than the current width of the upstream, we flag this as a bad link.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">The PCIe Error Logging Service independently runs on our hardware servers and independently records the rate of corrected and uncorrectable errors and their rates in a predetermined format (json).</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">MachineChecker checks for uncorrected errors. Even a single uncorrected error event qualifies a server as faulty.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">During its periodic run, MachineChecker also looks up the generated files on the servers and checks them against a prerecorded source of truth in </span><a href="https://research.fb.com/publications/holistic-configuration-management-at-facebook/"><span style="font-weight: 400;">Configerator</span></a><span style="font-weight: 400;"> (our configuration management system) for a threshold per platform. If the rate exceeds a preset threshold, the hardware is marked as faulty. These thresholds are easily adjustable per platform.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">We also leverage PCIcrawler, which is also preinstalled on all our hardware servers, to check for PCIe AER issues.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">We leverage our in-house tool’s knowledge of hardware configuration to associate a PCIe address to a given hardware part.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">MachineChecker uses PCIcrawler (for link width, link speed, and AER information) and the PCIe Error Parsing Service (which in turn uses SEL and dmesg) to identify hardware issues and create alerts or alarms. MachineChecker leverages information from our in-house tool to identify the hardware components associated with the PCIe addresses and assists data center operators (who may need to swap out the hardware) by supplying additional information, such as the component’s location, model information, and vendor name.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">Application production engineers can subscribe to these alerts or alarms and customize workflows for monitoring, alerting, remediation, and custom repair.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">A subset of all the alerts can undergo a particular remediation. We can also fine-tune the remediation and add special casing, restricting the remediation to, for example, a firmware upgrade if a particular case is well known.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">If the remediation fails sufficiently, a hardware repair ticket is automatically created so that the data center operators can swap the bad hardware component or server with a tested good one.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">We have rate limiting in several places as a safety net to prevent bugs in the code or mass draining and unprovisioning, which can result in service outages if not handled properly.</span></li>
|
||
</ol>
|
||
<p><span style="font-weight: 400;">We’ve added hardware health coverage and fixed several thousand servers and server components with this methodology. We continue to detect, diagnose, remediate, and repair hundreds of servers every week with various PCIe faults. This has made our hardware fleet more reliable, resilient, and performant.</span></p>
|
||
<p><i><span style="font-weight: 400;">We’d like to thank Aaron Miller, Aleksander Książek, Chris Chen, Deomid Ryabkov, Wren Turkal and many others who contributed to this work in different aspects.</span></i></p>
|
||
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-facebook"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-facebook-17580"
|
||
class="share-facebook sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/?share=facebook"
|
||
target="_blank"
|
||
aria-labelledby="sharing-facebook-17580"
|
||
>
|
||
<span id="sharing-facebook-17580" hidden>Share on Facebook (Opens in new window)</span>
|
||
<span>Facebook</span>
|
||
</a></li><li class="share-threads"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-threads-17580"
|
||
class="share-threads sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/?share=threads"
|
||
target="_blank"
|
||
aria-labelledby="sharing-threads-17580"
|
||
>
|
||
<span id="sharing-threads-17580" hidden>Share on Threads (Opens in new window)</span>
|
||
<span>Threads</span>
|
||
</a></li><li class="share-jetpack-whatsapp"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-whatsapp-17580"
|
||
class="share-jetpack-whatsapp sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/?share=jetpack-whatsapp"
|
||
target="_blank"
|
||
aria-labelledby="sharing-whatsapp-17580"
|
||
>
|
||
<span id="sharing-whatsapp-17580" hidden>Share on WhatsApp (Opens in new window)</span>
|
||
<span>WhatsApp</span>
|
||
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-linkedin-17580"
|
||
class="share-linkedin sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/?share=linkedin"
|
||
target="_blank"
|
||
aria-labelledby="sharing-linkedin-17580"
|
||
>
|
||
<span id="sharing-linkedin-17580" hidden>Share on LinkedIn (Opens in new window)</span>
|
||
<span>LinkedIn</span>
|
||
</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-reddit-17580"
|
||
class="share-reddit sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/?share=reddit"
|
||
target="_blank"
|
||
aria-labelledby="sharing-reddit-17580"
|
||
>
|
||
<span id="sharing-reddit-17580" hidden>Share on Reddit (Opens in new window)</span>
|
||
<span>Reddit</span>
|
||
</a></li><li class="share-x"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-x-17580"
|
||
class="share-x sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/?share=x"
|
||
target="_blank"
|
||
aria-labelledby="sharing-x-17580"
|
||
>
|
||
<span id="sharing-x-17580" hidden>Share on X (Opens in new window)</span>
|
||
<span>X</span>
|
||
</a></li><li class="share-bluesky"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-bluesky-17580"
|
||
class="share-bluesky sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/?share=bluesky"
|
||
target="_blank"
|
||
aria-labelledby="sharing-bluesky-17580"
|
||
>
|
||
<span id="sharing-bluesky-17580" hidden>Share on Bluesky (Opens in new window)</span>
|
||
<span>Bluesky</span>
|
||
</a></li><li class="share-mastodon"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-mastodon-17580"
|
||
class="share-mastodon sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/?share=mastodon"
|
||
target="_blank"
|
||
aria-labelledby="sharing-mastodon-17580"
|
||
>
|
||
<span id="sharing-mastodon-17580" hidden>Share on Mastodon (Opens in new window)</span>
|
||
<span>Mastodon</span>
|
||
</a></li><li class="share-custom share-custom-hackernews"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-custom-17580"
|
||
class="share-custom share-custom-hackernews sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/?share=custom-1699562127"
|
||
target="_blank"
|
||
aria-labelledby="sharing-custom-17580"
|
||
>
|
||
<span id="sharing-custom-17580" hidden>Share on Hacker News (Opens in new window)</span>
|
||
<span style="background-image:url("https://news.ycombinator.com/y18.svg");">Hacker News</span>
|
||
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-email-17580"
|
||
class="share-email sd-button share-icon"
|
||
href="mailto:?subject=%5BShared%20Post%5D%20How%20Facebook%20deals%20with%20PCIe%20faults%20to%20keep%20our%20data%20centers%20running%20reliably&body=https%3A%2F%2Fengineering.fb.com%2F2021%2F06%2F02%2Fdata-center-engineering%2Fhow-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably%2F&share=email"
|
||
target="_blank"
|
||
aria-labelledby="sharing-email-17580"
|
||
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you're having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="b93ab5550e" data-email-share-track-url="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/?share=email">
|
||
<span id="sharing-email-17580" hidden>Email a link to a friend (Opens in new window)</span>
|
||
<span>Email</span>
|
||
</a></li><li class="share-end"></li></ul></div></div></div>
|
||
|
||
</div><!-- .entry-content -->
|
||
|
||
|
||
</article><!-- #post-## -->
|
||
|
||
<footer class="entry-footer">
|
||
|
||
|
||
</footer><!-- .entry-footer -->
|
||
|
||
<div class="fb-like"
|
||
data-href="https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/"
|
||
data-width=""
|
||
data-layout="standard"
|
||
data-action="like"
|
||
data-size="small"
|
||
data-share="true">
|
||
</div>
|
||
|
||
|
||
<div class="related-articles">
|
||
<div class="related-articles__header">
|
||
<h3 class="related-articles__header__headline">Read More in Data Center Engineering</h3>
|
||
<a
|
||
aria-label="View all articles in this Data Center Engineering"
|
||
class="related-articles__header__link"
|
||
href="https://engineering.fb.com/category/data-center-engineering/"
|
||
>
|
||
View All </a>
|
||
</div>
|
||
|
||
<div class="article-grids related-articles__grid">
|
||
<div class="row">
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17580 type-post status-publish format-standard has-post-thumbnail hentry category-data-center-engineering category-production-engineering fb_content_type-article" id="post-24399">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/MetaRoCE-HERO-final.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-08-24T11:02:29-07:00">
|
||
AUG 24, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/metaroce-rdma-transport-ai-ethernet/" rel="bookmark" aria-label="Read More about MetaRoCE: A New RDMA Transport Built for AI-Scale Ethernet">
|
||
MetaRoCE: A New RDMA Transport Built for AI-Scale Ethernet </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17580 type-post status-publish format-standard has-post-thumbnail hentry category-data-center-engineering category-production-engineering fb_content_type-article" id="post-24150">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Metas-AI-Storage-Blueprint-at-Scale-Hero-1.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-07-01T09:00:36-07:00">
|
||
JUL 1, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/07/01/data-infrastructure/metas-ai-storage-blueprint-at-scale/" rel="bookmark" aria-label="Read More about Meta’s AI Storage Blueprint at Scale">
|
||
Meta’s AI Storage Blueprint at Scale </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17580 type-post status-publish format-standard has-post-thumbnail hentry category-data-center-engineering category-production-engineering fb_content_type-article" id="post-24022">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Validating-PowerLoss-Readiness-Meta-hero.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-06-03T10:00:44-07:00">
|
||
JUN 3, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/06/03/data-center-engineering/lights-out-systems-on-validating-instant-power-loss-readiness/" rel="bookmark" aria-label="Read More about Lights Out, Systems On: Validating Instant Power Loss Readiness">
|
||
Lights Out, Systems On: Validating Instant Power Loss Readiness </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17580 type-post status-publish format-standard has-post-thumbnail hentry category-data-center-engineering category-production-engineering fb_content_type-article" id="post-23769">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/03/Meta-AI-for-American-Concrete-HERO.jpg?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-03-30T09:00:34-07:00">
|
||
MAR 30, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/03/30/data-center-engineering/ai-for-american-produced-cement-and-concrete/" rel="bookmark" aria-label="Read More about AI for American-Produced Cement and Concrete">
|
||
AI for American-Produced Cement and Concrete </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17580 type-post status-publish format-standard has-post-thumbnail hentry category-data-center-engineering category-production-engineering fb_content_type-article" id="post-23617">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/02/rrcclx-innovating-gpu-communications-amd-platforms-meta-HERO.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-02-24T13:30:54-08:00">
|
||
FEB 24, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/02/24/data-center-engineering/rrcclx-innovating-gpu-communications-amd-platforms-meta/" rel="bookmark" aria-label="Read More about RCCLX: Innovating GPU Communications on AMD Platforms">
|
||
RCCLX: Innovating GPU Communications on AMD Platforms </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17580 type-post status-publish format-standard has-post-thumbnail hentry category-data-center-engineering category-production-engineering fb_content_type-article" id="post-23611">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/02/Meta-Prometheus-6.jpg?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-02-09T09:00:33-08:00">
|
||
FEB 9, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/02/09/data-center-engineering/building-prometheus-how-backend-aggregation-enables-gigawatt-scale-ai-clusters/" rel="bookmark" aria-label="Read More about Building Prometheus: How Backend Aggregation Enables Gigawatt-Scale AI Clusters">
|
||
Building Prometheus: How Backend Aggregation Enables Gigawatt-Scale AI Clusters </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
</div><!-- /.row -->
|
||
</div><!--/.article-grids -->
|
||
|
||
</div><!-- /.related-articles -->
|
||
|
||
|
||
<!-- include the carousel nav here. -->
|
||
|
||
|
||
|
||
|
||
</main><!-- #main -->
|
||
|
||
</div><!-- #primary -->
|
||
|
||
<!-- Do the right sidebar check. -->
|
||
|
||
|
||
|
||
|
||
<div class="col-md-4 widget-area" id="right-sidebar" role="complementary">
|
||
<aside id="fb_related_post_widget-4" class="widget widget_fb_related_post_widget"><h3 class="widget-title">Related Posts</h3><hr>
|
||
<div class="aside-related-projects"><div class="article-grid-redesign">
|
||
<article class="post-16435 post type-post status-publish has-post-thumbnail hentry" id="post-16435">
|
||
<a href="https://engineering.fb.com/2020/08/05/open-source/pcicrawler/" rel="bookmark" aria-label="Read More about Pcicrawler: A Python-based command-line interface tool to debug PCI issues at scale">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2020/08/ENG-blog-OSiB-dark-blue-hero.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/ENG-blog-OSiB-dark-blue-hero.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2020/08/ENG-blog-OSiB-dark-blue-hero.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Aug 05, 2020">Aug 05, 2020</time></span>
|
||
<h4 class="entry-title">Pcicrawler: A Python-based command-line interface tool to debug PCI issues at scale</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div><div class="article-grid-redesign">
|
||
<article class="post-17009 post type-post status-publish has-post-thumbnail hentry" id="post-17009">
|
||
<a href="https://engineering.fb.com/2020/12/09/data-center-engineering/how-facebook-keeps-its-large-scale-infrastructure-hardware-up-and-running/" rel="bookmark" aria-label="Read More about How Facebook keeps its large-scale infrastructure hardware up and running">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2020/12/Prineville_Server-room-dark2-e1607617109406.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2020/12/Prineville_Server-room-dark2-e1607617109406.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2020/12/Prineville_Server-room-dark2-e1607617109406.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Dec 09, 2020">Dec 09, 2020</time></span>
|
||
<h4 class="entry-title">How Facebook keeps its large-scale infrastructure hardware up and running</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div><div class="article-grid-redesign">
|
||
<article class="post-7735 post type-post status-publish has-post-thumbnail hentry" id="post-7735">
|
||
<a href="https://engineering.fb.com/2016/07/11/production-engineering/making-facebook-self-healing-automating-proactive-rack-maintenance/" rel="bookmark" aria-label="Read More about Making Facebook self-healing: Automating proactive rack maintenance">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2016/07/gine0adj0smwglogandf-iiaaaaabj0jaaab.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2016/07/gine0adj0smwglogandf-iiaaaaabj0jaaab.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2016/07/gine0adj0smwglogandf-iiaaaaabj0jaaab.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Jul 11, 2016">Jul 11, 2016</time></span>
|
||
<h4 class="entry-title">Making Facebook self-healing: Automating proactive rack maintenance</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div></div></aside><aside id="wpb_widget-3" class="widget widget_wpb_widget"><h3 class="widget-title">Related Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2278965319592702/>
|
||
<div class="job-description">
|
||
Data Governance Lead, Infrastructure
|
||
</div>
|
||
<div class="job-location">
|
||
BELLEVUE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2278965319592702/>
|
||
<div class="job-description">
|
||
Data Governance Lead, Infrastructure
|
||
</div>
|
||
<div class="job-location">
|
||
MENLO PARK, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2278965319592702/>
|
||
<div class="job-description">
|
||
Data Governance Lead, Infrastructure
|
||
</div>
|
||
<div class="job-location">
|
||
SEATTLE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1458825126087114/>
|
||
<div class="job-description">
|
||
Generator Engineer, Data Center Infrastructure
|
||
</div>
|
||
<div class="job-location">
|
||
RESTON, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1458825126087114/>
|
||
<div class="job-description">
|
||
Generator Engineer, Data Center Infrastructure
|
||
</div>
|
||
<div class="job-location">
|
||
MENLO PARK, US
|
||
</div>
|
||
</a>
|
||
</li> </ul>
|
||
<a href="https://www.metacareers.com">
|
||
<div class="btn available-positions-btn">
|
||
See All Jobs </div>
|
||
</a>
|
||
</aside>
|
||
</div><!-- #secondary -->
|
||
|
||
|
||
|
||
<!-- <php get_template_part('content-single-side-bar'); ?> -->
|
||
|
||
<!-- Do the right sidebar check. -->
|
||
|
||
</div><!-- Container end -->
|
||
|
||
</div><!-- Wrapper end -->
|
||
|
||
</div>
|
||
|
||
<!-- ******************* The Footer Full-width Widget Area ******************* -->
|
||
|
||
<div class="wrapper wrapper-community-outer">
|
||
<div class="wrapper-community">
|
||
|
||
<div class="community" tabindex="-1">
|
||
|
||
<div class="row">
|
||
<!-- community section is generated through WordPress widget -->
|
||
<div id="wpb_widget-2" class="footer-widget widget_wpb_widget widget-count-3 col-md-4"><h3 class="widget-title">Available Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2278965319592702/>
|
||
<div class="job-description">
|
||
Data Governance Lead, Infrastructure
|
||
</div>
|
||
<div class="job-location">
|
||
BELLEVUE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2278965319592702/>
|
||
<div class="job-description">
|
||
Data Governance Lead, Infrastructure
|
||
</div>
|
||
<div class="job-location">
|
||
MENLO PARK, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2278965319592702/>
|
||
<div class="job-description">
|
||
Data Governance Lead, Infrastructure
|
||
</div>
|
||
<div class="job-location">
|
||
SEATTLE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1458825126087114/>
|
||
<div class="job-description">
|
||
Generator Engineer, Data Center Infrastructure
|
||
</div>
|
||
<div class="job-location">
|
||
RESTON, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1458825126087114/>
|
||
<div class="job-description">
|
||
Generator Engineer, Data Center Infrastructure
|
||
</div>
|
||
<div class="job-location">
|
||
MENLO PARK, US
|
||
</div>
|
||
</a>
|
||
</li> </ul>
|
||
<a href="https://www.metacareers.com">
|
||
<div class="btn available-positions-btn">
|
||
See All Jobs </div>
|
||
</a>
|
||
</div><!-- .footer-widget --><div id="custom_html-2" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Technology at Meta</h3><div class="textwidget custom-html-widget"><ul class="stay-connected-lists">
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-fb-engineering" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Engineering at Meta - X
|
||
</div>
|
||
<a onclick='return twitterFollow("Meta_Engineers")'>
|
||
<button class="btn stay-connected-btn">
|
||
Follow
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-AI" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
AI at Meta
|
||
</div>
|
||
<a href="https://ai.meta.com/blog/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta Quest Blog
|
||
</div>
|
||
<a href="https://www.meta.com/blog/quest/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta for Developers
|
||
</div>
|
||
<a href="https://developers.facebook.com/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-bug-bounty" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta Bug Bounty
|
||
</div>
|
||
<a href="https://bugbounty.meta.com/">
|
||
<button class="btn stay-connected-btn">
|
||
Learn more
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
|
||
<li class="stay-connected-item">
|
||
<img class="stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/rss.png" alt="footer-rss" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
RSS
|
||
</div>
|
||
<a href="https://code.facebook.com/posts/rss/">
|
||
<button class="btn stay-connected-btn">
|
||
Subscribe
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div></div><!-- .footer-widget --><div id="custom_html-3" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Open Source</h3><div class="textwidget custom-html-widget"><p class="opensource-desc">
|
||
Meta believes in building community through open source technology. Explore our latest projects in Artificial Intelligence, Data Infrastructure, Development Tools, Front End, Languages, Platforms, Security, Virtual Reality, and more.
|
||
</p>
|
||
<ul class="opensource-cats">
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/android.png" alt="android" loading="lazy" width="50" height="59">
|
||
<div class="opensource-cap">
|
||
ANDROID
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/ios.png" alt="ios" loading="lazy" width="48" height="58">
|
||
<div class="opensource-cap">
|
||
iOS
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/web.png" alt="web" loading="lazy" width="55" height="49">
|
||
<div class="opensource-cap">
|
||
WEB
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/backend.png" alt="backend" loading="lazy" width="56" height="51">
|
||
<div class="opensource-cap">
|
||
BACKEND
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/hardware.png" alt="hardware" loading="lazy" width="55" height="44">
|
||
<div class="opensource-cap">
|
||
HARDWARE
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
|
||
<button class="btn opensource-btn" onclick="window.location.href='https://opensource.fb.com/'">
|
||
Learn More
|
||
</button>
|
||
|
||
</div></div><!-- .footer-widget -->
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</div><!-- #wrapper-footer-full -->
|
||
|
||
|
||
<div class="wrapper-footer" id="wrapper-footer">
|
||
<div class="inner-container">
|
||
<div class="footer">
|
||
<div class="row">
|
||
<div class="col-md-12">
|
||
<footer class="meta-footer" id="meta-footer">
|
||
<div class="site-info row justify-content-between">
|
||
<div class="col-md-5 meta-logo-container">
|
||
<a href="https://about.facebook.com/">
|
||
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/meta_logo_full.svg" width="89" height="18" alt="Meta" />
|
||
</a>
|
||
<div id="text-2" class="widget widget_text"> <div class="textwidget"><p>Engineering at Meta is a technical news resource for engineers interested in how we solve large-scale technical challenges at Meta.</p>
|
||
</div>
|
||
</div> </div>
|
||
<div class="col-md-6 footer-meta-links-container">
|
||
<ul class="meta-footer-links">
|
||
<li class="footer-link"><a href="https://engineering.fb.com">Home</a></li><li class="footer-link"><a href="https://about.meta.com/">Company Info</a></li><li class="footer-link"><a href="https://www.metacareers.com/?ref=engineering.fb.com">Careers</a></li> </ul>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="footer">
|
||
<div class="row">
|
||
|
||
<div class="col-md-12">
|
||
|
||
<footer class="site-footer" id="colophon">
|
||
<div class="inner-container">
|
||
<div class="site-info row justify-content-between">
|
||
|
||
<div class="col-md-4 copyright-container">
|
||
<div class="copyright"> © 2026 Meta</div>
|
||
</div>
|
||
|
||
<!-- static footer -->
|
||
<div class="col-md-7 footer-links-container">
|
||
<ul class="footer-links">
|
||
<li class="footer-link"><a href="https://www.facebook.com/policies">Terms</a></li><li class="footer-link"><a href="https://www.facebook.com/privacy/policy">Privacy</a></li><li class="footer-link"><a href="/privacy">Cookies</a></li><li class="footer-link"><a href="https://www.facebook.com/help">Help</a></li> </ul>
|
||
</div>
|
||
</div>
|
||
</div><!-- .site-info -->
|
||
|
||
</footer><!-- #colophon -->
|
||
|
||
</div><!--col end -->
|
||
|
||
</div><!-- row end -->
|
||
|
||
</div><!-- //.inner-container -->
|
||
</div><!-- wrapper end -->
|
||
|
||
</div><!-- #page we need this extra closing tag here -->
|
||
|
||
<script type="speculationrules">
|
||
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/code-fb-com/*","/wp-content/themes/understrap/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
|
||
</script>
|
||
<div class="gdprconsent-container" id="GDPRConsentBar">
|
||
<div class="gdprconsent-wrapper">
|
||
<div class="gdprconsent-content">
|
||
<p>To help personalize content, tailor and measure ads and provide a safer experience, we use cookies. By clicking or navigating the site, you agree to allow our collection of information on and off Facebook through cookies. Learn more, including about available controls: <a href="/privacy">Cookie Policy</a></p>
|
||
<div class="gdprconsent-button">
|
||
<button type="button" data-gdpr-accept>Accept</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Twitter universal website tag code -->
|
||
<script>
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
|
||
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
|
||
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
|
||
// Insert Twitter Pixel ID and Standard Event data below
|
||
twq('init','o31u0');
|
||
twq('track','PageView');
|
||
});
|
||
}
|
||
</script>
|
||
<!-- End Twitter universal website tag code -->
|
||
|
||
|
||
<script type="text/javascript">
|
||
window.WPCOM_sharing_counts = {"https://engineering.fb.com/2021/06/02/data-center-engineering/how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably/":17580};
|
||
</script>
|
||
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KZRLGZ8" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
<noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=660742068067366&ev=PageView&noscript=1"/></noscript>
|
||
<script id="child-understrap-scripts-js-extra">
|
||
var loadmore_params = {"restfulURL":"https://engineering.fb.com/wp-json/fb/v1/loadmore","searchText":"","posts":"{\"page\":0,\"year\":2021,\"monthnum\":6,\"day\":2,\"name\":\"how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\",\"category_name\":\"data-center-engineering\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","post_type":"post"};
|
||
var search_params = {"ajaxurl":"https://engineering.fb.com/wp-admin/admin-ajax.php","posts":"{\"page\":0,\"year\":2021,\"monthnum\":6,\"day\":2,\"name\":\"how-facebook-deals-with-pcie-faults-to-keep-our-data-centers-running-reliably\",\"category_name\":\"data-center-engineering\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","article_filter":"2f23ad5ce3"};
|
||
var more_articles = {"next_article":{"label":"Next","permalink":"https://engineering.fb.com/2021/06/15/data-infrastructure/network-hose/","title":"Network hose: Managing uncertain network demand with model simplicity","thumbnail":["https://engineering.fb.com/wp-content/uploads/2021/06/networkhose_hero.jpg?w=580&h=326&crop=1",580,326,true]},"prev_article":{"label":"Prev","permalink":"https://engineering.fb.com/2021/05/20/networking-traffic/peering-automation/","title":"Peering automation at Facebook","thumbnail":["https://engineering.fb.com/wp-content/uploads/2021/05/Peering_hero.jpg?w=580&h=326&crop=1",580,326,true]}};
|
||
//# sourceURL=child-understrap-scripts-js-extra
|
||
</script>
|
||
<script type="text/javascript" src="https://engineering.fb.com/_static/??/wp-content/themes/code-fb-com/dist/js/child-theme.min.js,/wp-content/themes/code-fb-com/js/prism.js?m=1783017802j" ></script><script id="sharing-js-js-extra">
|
||
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":""};
|
||
//# sourceURL=sharing-js-js-extra
|
||
</script>
|
||
<script id="sharing-js-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/_inc/build/sharedaddy/sharing.min.js?ver=16.1.3"></script>
|
||
<script id="sharing-js-js-after">
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-facebook' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-threads' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-threads' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomthreads', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-linkedin' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-x' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-x' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomx', 'menubar=1,resizable=1,width=600,height=350' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-bluesky' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-bluesky' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcombluesky', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-mastodon' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-mastodon' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcommastodon', 'menubar=1,resizable=1,width=460,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
//# sourceURL=sharing-js-js-after
|
||
</script>
|
||
<script id="video-analytics-js-after">
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
let youtube = document.querySelectorAll('[src*="youtube-nocookie"]');
|
||
for (let i = 0; i < youtube.length; i++) {
|
||
youtube[i].src = youtube[i].src.replace('youtube-nocookie.com', 'youtube.com');
|
||
}
|
||
let vimeo = document.querySelectorAll('[src*="vimeo"]');
|
||
for (let i = 0; i < vimeo.length; i++) {
|
||
vimeo[i].src = vimeo[i].src.replace('&dnt=true', '');
|
||
}
|
||
});
|
||
}
|
||
//# sourceURL=video-analytics-js-after
|
||
</script>
|
||
<script id="wp-emoji-settings" type="application/json">
|
||
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://engineering.fb.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
|
||
</script>
|
||
<script type="module">
|
||
/*! This file is auto-generated */
|
||
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
|
||
//# sourceURL=https://engineering.fb.com/wp-includes/js/wp-emoji-loader.min.js
|
||
</script>
|
||
|
||
</body>
|
||
|
||
</html>
|