1272 lines
106 KiB
HTML
1272 lines
106 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en-US">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||
<meta name="mobile-web-app-capable" content="yes">
|
||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||
<meta name="apple-mobile-web-app-title" content="Engineering at Meta - Engineering at Meta Blog">
|
||
<meta property="fb:app_id" content="1425766027653270" />
|
||
<link rel="profile" href="http://gmpg.org/xfn/11">
|
||
<link rel="pingback" href="https://engineering.fb.com/xmlrpc.php">
|
||
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
|
||
<script async src="https://engineering.fb.com/wp-content/themes/code-fb-com/dist/js/optimize.min.js"></script>
|
||
|
||
<!-- This site is optimized with the Yoast SEO Premium plugin v19.3 (Yoast SEO v27.3) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
|
||
<title>Minesweeper automates root cause analysis as a first-line defense against bugs</title>
|
||
<meta name="description" content="Minesweeper is a new technique for automating root cause analysis (RCA) that identifies the causes of bugs based on their symptoms." />
|
||
<link rel="canonical" href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/" />
|
||
<meta name="twitter:label1" content="Written by" />
|
||
<meta name="twitter:data1" content="Vijay Murali, Edward Yao, Umang Mathur, Satish Chandra" />
|
||
<meta name="twitter:label2" content="Est. reading time" />
|
||
<meta name="twitter:data2" content="6 minutes" />
|
||
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/#article","isPartOf":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/"},"author":[{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/0200be4a4910783b53becfef3450a13b"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/72bc3e80000b88b1ce5d6b6335dc7c10"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/b77df78946a6fec9a9d391e872c4376d"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/5df6c2d419558e6d738358e11b61bc40"}],"headline":"Minesweeper automates root cause analysis as a first-line defense against bugs","datePublished":"2021-02-09T17:00:06+00:00","dateModified":"2021-02-12T05:49:30+00:00","mainEntityOfPage":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/"},"wordCount":1075,"publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"image":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/02\/Minesweeper-Hero.png","articleSection":["DevInfra","Networking & Traffic","Production Engineering","Security & Privacy"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/","url":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/","name":"Minesweeper automates root cause analysis as a first-line defense against bugs","isPartOf":{"@id":"https:\/\/engineering.fb.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/#primaryimage"},"image":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/02\/Minesweeper-Hero.png","datePublished":"2021-02-09T17:00:06+00:00","dateModified":"2021-02-12T05:49:30+00:00","description":"Minesweeper is a new technique for automating root cause analysis (RCA) that identifies the causes of bugs based on their symptoms.","breadcrumb":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/#primaryimage","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/02\/Minesweeper-Hero.png","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/02\/Minesweeper-Hero.png","width":1920,"height":1080,"caption":"Minesweeper"},{"@type":"BreadcrumbList","@id":"https:\/\/engineering.fb.com\/2021\/02\/09\/developer-tools\/minesweeper\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/engineering.fb.com\/"},{"@type":"ListItem","position":2,"name":"Minesweeper automates root cause analysis as a first-line defense against bugs"}]},{"@type":"WebSite","@id":"https:\/\/engineering.fb.com\/#website","url":"https:\/\/engineering.fb.com\/","name":"Engineering at Meta","description":"Engineering at Meta Blog","publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/engineering.fb.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/engineering.fb.com\/#organization","name":"Meta","url":"https:\/\/engineering.fb.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","width":29011,"height":12501,"caption":"Meta"},"image":{"@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/Engineering\/","https:\/\/x.com\/fb_engineering"]},[],{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/0200be4a4910783b53becfef3450a13b","name":"Vijay Murali","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Vijay Murali"},"url":"https:\/\/engineering.fb.com\/author\/vijay-murali\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/72bc3e80000b88b1ce5d6b6335dc7c10","name":"Edward Yao","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Edward Yao"},"url":"https:\/\/engineering.fb.com\/author\/edward-yao\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/b77df78946a6fec9a9d391e872c4376d","name":"Umang Mathur","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Umang Mathur"},"url":"https:\/\/engineering.fb.com\/author\/umang-mathur\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/5df6c2d419558e6d738358e11b61bc40","name":"Satish Chandra","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Satish Chandra"},"description":"Software Engineer","url":"https:\/\/engineering.fb.com\/author\/satish-chandra\/"}]}</script>
|
||
<!-- / Yoast SEO Premium plugin. -->
|
||
|
||
|
||
<link rel='dns-prefetch' href='//secure.gravatar.com' />
|
||
<link rel='dns-prefetch' href='//v0.wordpress.com' />
|
||
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta » Feed" href="https://engineering.fb.com/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta » Comments Feed" href="https://engineering.fb.com/comments/feed/" />
|
||
<script async src="https://www.googleoptimize.com/optimize.js?id=OPT-TPNDQC5" onerror="dataLayer.hide.end && dataLayer.hide.end()"></script>
|
||
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2021%2F02%2F09%2Fdeveloper-tools%2Fminesweeper%2F" />
|
||
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2021%2F02%2F09%2Fdeveloper-tools%2Fminesweeper%2F&format=xml" />
|
||
<style id="wp-img-auto-sizes-contain-inline-css">
|
||
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
|
||
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
|
||
</style>
|
||
<style id="wp-emoji-styles-inline-css">
|
||
|
||
img.wp-smiley, img.emoji {
|
||
display: inline !important;
|
||
border: none !important;
|
||
box-shadow: none !important;
|
||
height: 1em !important;
|
||
width: 1em !important;
|
||
margin: 0 0.07em !important;
|
||
vertical-align: -0.1em !important;
|
||
background: none !important;
|
||
padding: 0 !important;
|
||
}
|
||
/*# sourceURL=wp-emoji-styles-inline-css */
|
||
</style>
|
||
<style id="wp-block-library-inline-css">
|
||
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
|
||
|
||
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
|
||
</style>
|
||
<style id="classic-theme-styles-inline-css">
|
||
/*! This file is auto-generated */
|
||
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
|
||
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
|
||
</style>
|
||
|
||
<style id="global-styles-inline-css">
|
||
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
|
||
/*# sourceURL=global-styles-inline-css */
|
||
</style>
|
||
|
||
<link rel='stylesheet' id='all-css-12' href='https://engineering.fb.com/_static/??-eJyNjUsOwjAMRC9EaiqghQXiKFVqhzSQn+pEiNuTFJBAbNjZTzNv4BYFBp+UTxBt1sYzFKQpzpVz5WQ4QSUv0CDzCj6KaVJOMWAgJc5jwe7ZKTnAyVgSS6Jxxv90XRbv3YtKUeJVtF3TgguUbZHyJGdFkui+nMbrvxWD8QhjLvvAAY20wgYd+OupspM7tv3+sO533Wb7AImQY4U=' type='text/css' media='all' />
|
||
<script id="gdprconsent-js" src="https://engineering.fb.com/wp-content/plugins/wp-gdpr-consent/dist/gdprconsent.js?ver=2.0.0"></script>
|
||
<script id="gdprconsent-js-after">
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack('https://www.googletagmanager.com/gtag/js?id=391063774');
|
||
window.gdprSafeTrack(function() {
|
||
var google_analytics_g4 = decodeURIComponent( '391063774' );
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
gtag('js', new Date());
|
||
gtag('config', google_analytics_g4);
|
||
});
|
||
}
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
var google_tag_manager = decodeURIComponent( 'GTM-KZRLGZ8' );
|
||
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||
})(window,document,'script','dataLayer',google_tag_manager);
|
||
});
|
||
}
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
var facebook_pixel = decodeURIComponent( '660742068067366' );
|
||
!function(f,b,e,v,n,t,s)
|
||
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
|
||
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
|
||
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
|
||
n.queue=[];t=b.createElement(e);t.async=!0;
|
||
t.src=v;s=b.getElementsByTagName(e)[0];
|
||
s.parentNode.insertBefore(t,s)}(window, document,'script',
|
||
'https://connect.facebook.net/en_US/fbevents.js');
|
||
fbq('init', facebook_pixel);
|
||
fbq('track', 'PageView');
|
||
});
|
||
}
|
||
//# sourceURL=gdprconsent-js-after
|
||
</script>
|
||
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJzTLy/QzcxLzilNSS3WzwKiwtLUokoopZebmaeXVayjj0+Rbm5melFiSSpUsX2uraG5hbmZgYmloUkWAK+3Iic=" ></script><script type="text/javascript" src="https://engineering.fb.com/wp-content/themes/understrap/js/popper.min.js?m=1773259670g" ></script><link rel="https://api.w.org/" href="https://engineering.fb.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.fb.com/wp-json/wp/v2/posts/17180" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.fb.com/xmlrpc.php?rsd" />
|
||
<meta name="generator" content="WordPress 7.1" />
|
||
<link rel='shortlink' href='https://wp.me/pa0Lhq-4t6' />
|
||
<link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/Optimistic_Display_W_Md.woff2" as="font" type="font/woff2" crossorigin="anonymous">
|
||
<!-- Jetpack Open Graph Tags -->
|
||
<meta property="og:type" content="article" />
|
||
<meta property="og:title" content="Minesweeper automates root cause analysis as a first-line defense against bugs" />
|
||
<meta property="og:url" content="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/" />
|
||
<meta property="og:description" content="Root cause analysis (RCA) is an important part of fixing any bug. After all, you can’t solve a problem without getting to the heart of it. But RCA isn’t always simple, especially at a scale like Fa…" />
|
||
<meta property="article:published_time" content="2021-02-09T17:00:06+00:00" />
|
||
<meta property="article:modified_time" content="2021-02-12T05:49:30+00:00" />
|
||
<meta property="og:site_name" content="Engineering at Meta" />
|
||
<meta property="og:image" content="https://engineering.fb.com/wp-content/uploads/2021/02/Minesweeper-Hero.png" />
|
||
<meta property="og:image:width" content="1920" />
|
||
<meta property="og:image:height" content="1080" />
|
||
<meta property="og:image:alt" content="Minesweeper" />
|
||
<meta property="og:locale" content="en_US" />
|
||
<meta name="twitter:text:title" content="Minesweeper automates root cause analysis as a first-line defense against bugs" />
|
||
<meta name="twitter:image" content="https://engineering.fb.com/wp-content/uploads/2021/02/Minesweeper-Hero.png?w=640" />
|
||
<meta name="twitter:image:alt" content="Minesweeper" />
|
||
<meta name="twitter:card" content="summary_large_image" />
|
||
|
||
<!-- End Jetpack Open Graph Tags -->
|
||
<style id="wp-custom-css">
|
||
.table-of-contents {
|
||
position: sticky;
|
||
top: 20px;
|
||
background: #fff;
|
||
padding: 1em;
|
||
border: 1px solid #ddd;
|
||
border-radius: 8px;
|
||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||
max-width: 250px;
|
||
}
|
||
</style>
|
||
|
||
<link rel="shortcut icon" href="https://engineering.fb.com/wp-content/themes/code-fb-com/favicon.ico" />
|
||
</head>
|
||
|
||
<body class="wp-singular post-template-default single single-post postid-17180 single-format-standard wp-theme-understrap wp-child-theme-code-fb-com group-blog">
|
||
|
||
<div class="hfeed site" id="page">
|
||
|
||
<!-- ******************* The Navbar Area ******************* -->
|
||
<div class="wrapper-fluid wrapper-navbar" id="wrapper-navbar">
|
||
<a class="skip-link screen-reader-text sr-only" href="#content">
|
||
Skip to content </a>
|
||
|
||
<nav class="navbar navbar-expand-std">
|
||
|
||
<div class="nav-container">
|
||
|
||
|
||
<!-- Your site title as branding in the menu -->
|
||
|
||
<div class="nav-top-row">
|
||
|
||
<h1 class="navbar-brand mb-0">
|
||
<a rel="home" href="https://engineering.fb.com/" title="Engineering at Meta">
|
||
<img class="navbar-brand-logo" src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/logo-meta.svg" class="navbar-brand-logo" alt="Engineering at Meta" width="215" height="40" loading="lazy">
|
||
</a>
|
||
</h1>
|
||
|
||
<!-- end custom logo -->
|
||
<label for="search" class="screen-reader-text">Search this site</label>
|
||
<input type="text" id="search" name="search" class="form-control-lg navbar-search" autocomplete="off">
|
||
|
||
<button class="navbar-toggler navbar-searchbar-toggler"
|
||
type="button"
|
||
data-toggle="collapse"
|
||
data-target="#searchbarDropdown"
|
||
aria-controls="searchbarDropdown"
|
||
aria-expanded="false"
|
||
aria-label="Toggle navigation">
|
||
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/icon-search.svg" class="navbar-mobile-search-icon" loading="lazy" width="24" height="25" />
|
||
</button>
|
||
|
||
<!-- desktop language menu (mobile language menu is implemented via jQuery)-->
|
||
<div class="notranslate nice-select navbar-language form-control">
|
||
</div>
|
||
|
||
<button class="navbar-toggler collapsed navbar-menu-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
|
||
</div>
|
||
|
||
<div class="navbar-border"></div>
|
||
|
||
<!-- The WordPress Menu goes here -->
|
||
<div id="navbarNavDropdown" class="collapse navbar-collapse"><ul id="main-menu" class="navbar-nav"><li id="menu-item-60" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-60 dropdown"><a title="Open Source" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Open Source <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-12330" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-12330"><a title="Open Source" href="https://engineering.fb.com/category/open-source/" class="nav-link">Open Source</a></li>
|
||
<li id="menu-item-12331" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-12331"><a title="Meta Open Source" href="https://opensource.fb.com" class="nav-link">Meta Open Source</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2686" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2686 dropdown"><a title="Platforms" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Platforms <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-65" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-65"><a title="Android" href="https://engineering.fb.com/category/android/" class="nav-link">Android</a></li>
|
||
<li id="menu-item-66" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-66"><a title="iOS" href="https://engineering.fb.com/category/ios/" class="nav-link">iOS</a></li>
|
||
<li id="menu-item-67" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-67"><a title="Web" href="https://engineering.fb.com/category/web/" class="nav-link">Web</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2682" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2682 dropdown"><a title="Infrastructure Systems" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Infrastructure Systems <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-218" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-218"><a title="Core Infra" href="https://engineering.fb.com/category/core-infra/" class="nav-link">Core Infra</a></li>
|
||
<li id="menu-item-219" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-219"><a title="Data Infrastructure" href="https://engineering.fb.com/category/data-infrastructure/" class="nav-link">Data Infrastructure</a></li>
|
||
<li id="menu-item-220" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-220"><a title="DevInfra" href="https://engineering.fb.com/category/developer-tools/" class="nav-link">DevInfra</a></li>
|
||
<li id="menu-item-226" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-226"><a title="Production Engineering" href="https://engineering.fb.com/category/production-engineering/" class="nav-link">Production Engineering</a></li>
|
||
<li id="menu-item-227" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-227"><a title="Security & Privacy" href="https://engineering.fb.com/category/security/" class="nav-link">Security & Privacy</a></li>
|
||
<li id="menu-item-18607" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18607"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/systems-infrastructure/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2683" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2683 dropdown"><a title="Physical Infrastructure" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Physical Infrastructure <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-215" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-215"><a title="Connectivity" href="https://engineering.fb.com/category/connectivity/" class="nav-link">Connectivity</a></li>
|
||
<li id="menu-item-216" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-216"><a title="Data Center Engineering" href="https://engineering.fb.com/category/data-center-engineering/" class="nav-link">Data Center Engineering</a></li>
|
||
<li id="menu-item-217" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-217"><a title="Networking & Traffic" href="https://engineering.fb.com/category/networking-traffic/" class="nav-link">Networking & Traffic</a></li>
|
||
<li id="menu-item-18608" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18608"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/networking-connectivity/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2684" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2684 dropdown"><a title="Video Engineering & AR/VR" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Video Engineering & AR/VR <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-225" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-225"><a title="Video Engineering" href="https://engineering.fb.com/category/video-engineering/" class="nav-link">Video Engineering</a></li>
|
||
<li id="menu-item-223" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-223"><a title="Virtual Reality" href="https://engineering.fb.com/category/virtual-reality/" class="nav-link">Virtual Reality</a></li>
|
||
<li id="menu-item-13388" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13388"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/augmented-reality-virtual-reality/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2685" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2685 dropdown"><a title="Artificial Intelligence" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Artificial Intelligence <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-222" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-222"><a title="ML Applications" href="https://engineering.fb.com/category/ml-applications/" class="nav-link">ML Applications</a></li>
|
||
<li id="menu-item-221" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-221"><a title="AI Research" href="https://engineering.fb.com/category/ai-research/" class="nav-link">AI Research</a></li>
|
||
<li id="menu-item-13387" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13387"><a title="Research Publications" href="https://ai.facebook.com/results/?content_types%5B0%5D=publication" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-228" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-228"><a title="Watch Videos" href="/videos" class="nav-link">Watch Videos</a></li>
|
||
</ul></div> </div><!-- .container -->
|
||
|
||
<div class="search-container collapse" id="searchbarDropdown">
|
||
<div class="searchbar-search">
|
||
<div class="searchbar-container">
|
||
|
||
<div class="navbar-mobile-search-margin"></div>
|
||
<input type="text" name="search" placeholder="Search" class="form-control-lg navbar-mobile-search" autofocus autocomplete="off">
|
||
<button class="searchbar-clear">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</nav><!-- .site-navigation -->
|
||
|
||
</div><!-- .wrapper-navbar end -->
|
||
|
||
<div class="wrapper" id="single-wrapper">
|
||
|
||
<div class="single-container row" id="content" tabindex="-1">
|
||
|
||
<!-- Do the left sidebar check. -->
|
||
|
||
|
||
|
||
<div class="content-area" id="primary">
|
||
<main class="site-main" id="main">
|
||
|
||
|
||
|
||
<article class="post-17180 post type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools category-networking-traffic category-production-engineering category-security category-uncategorized" id="post-17180">
|
||
|
||
<header class="entry-header">
|
||
<div class="entry-meta">
|
||
POSTED ON <span class="posted-on entry-date"><time class="published updated" datetime="2021-02-09">FEBRUARY 9, 2021</time></span> TO <span class="cat-links"><a href="https://engineering.fb.com/category/developer-tools/" rel="category tag">DevInfra</a>, <a href="https://engineering.fb.com/category/networking-traffic/" rel="category tag">Networking & Traffic</a>, <a href="https://engineering.fb.com/category/production-engineering/" rel="category tag">Production Engineering</a>, <a href="https://engineering.fb.com/category/security/" rel="category tag">Security & Privacy</a>, <a href="https://engineering.fb.com/category/uncategorized/" rel="category tag">Uncategorized</a></span>
|
||
<h1 class="entry-title"><div>Minesweeper automates root cause analysis as a first-line defense against bugs</div></h1> </div><!-- .entry-meta -->
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
<figure id="post-feat-image-container">
|
||
<img width="1920" height="1080" src="https://engineering.fb.com/wp-content/uploads/2021/02/Minesweeper-Hero.png" class="wp-post-image" alt="Minesweeper" title="Minesweeper Hero" /> </figure>
|
||
|
||
<div class="entry-authors-social">
|
||
<div class="entry-authors">
|
||
By <a href="https://engineering.fb.com/author/vijay-murali/" title="Posts by Vijay Murali" class="author url fn" rel="author">Vijay Murali</a>, <a href="https://engineering.fb.com/author/edward-yao/" title="Posts by Edward Yao" class="author url fn" rel="author">Edward Yao</a>, <a href="https://engineering.fb.com/author/umang-mathur/" title="Posts by Umang Mathur" class="author url fn" rel="author">Umang Mathur</a>, <a href="https://engineering.fb.com/author/satish-chandra/" title="Posts by Satish Chandra" class="author url fn" rel="author">Satish Chandra</a> </div>
|
||
</div>
|
||
|
||
|
||
<div class="entry-content">
|
||
|
||
<p><span style="font-weight: 400;">Root cause analysis (RCA) is an important part of fixing any bug. After all, you can’t solve a problem without getting to the heart of it. But RCA <a href="https://engineering.fb.com/2019/11/08/developer-tools/fast-dimensional-analysis/">isn’t always simple</a>, especially at a scale like Facebook’s. When billions of people are using an app on a variety of platforms and devices, a single bug can create several different issues on its own and multiple bugs can happen simultaneously.</span></p>
|
||
<p><span style="font-weight: 400;">There was a time when on-call engineers had to spend hours, or even days, manually combing through error reports, looking for patterns to help them debug. Today, they can use</span> <span style="font-weight: 400;">Minesweeper</span><span style="font-weight: 400;"> — <a href="https://arxiv.org/abs/2010.09974">a technique we’ve developed</a> for automating RCA that identifies the causes of bugs based on their symptoms.</span></p>
|
||
<p><span style="font-weight: 400;">Minesweeper-based RCA is completely automated and scalable, and it’s grounded in formal statistical concepts. Our own evaluations of Minesweeper using real-world bug reports from Facebook’s apps have proven that it can perform RCA for tens of thousands of reports in minutes and can identify the root cause of bugs with 85 percent accuracy.</span></p>
|
||
<p><span style="font-weight: 400;">Since it was developed, Minesweeper has become Facebook’s first line of defense against bugs and has helped us prevent potentially wide-scale disruptions from affecting people on our apps.</span></p>
|
||
<h2><span style="font-weight: 400;">How does Minesweeper work?</span></h2>
|
||
<p><span style="font-weight: 400;">Anytime someone reports a bug through the Facebook app, the <a href="https://engineering.fb.com/2020/12/09/data-center-engineering/how-facebook-keeps-its-large-scale-infrastructure-hardware-up-and-running/">error reporting system</a> typically captures a chronological trace of actions (or “events”) that person performed in the app prior to encountering the bug. The idea is to get a snapshot of what might have caused the error to happen, such as the example below:</span></p>
|
||
<p><img decoding="async" class="alignnone size-full wp-image-17181" src="https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_1.png" alt="" width="1262" height="120" srcset="https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_1.png 1262w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_1.png?resize=916,87 916w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_1.png?resize=768,73 768w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_1.png?resize=1024,97 1024w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_1.png?resize=96,9 96w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_1.png?resize=192,18 192w" sizes="(max-width: 992px) 100vw, 62vw" /></p>
|
||
<p><span style="font-weight: 400;">Minesweeper scans these traces to look for distinctive patterns that could point to the cause of a bug. Traces that contain the bug (the test group) are compared with traces that do not (the control group). Minesweeper finds patterns of events that are statistically distinctive to the test group as opposed to the control group. These patterns are likely to be correlated to the bug and can thereby point toward its root cause.</span></p>
|
||
<p><span style="font-weight: 400;">Here’s an example. Suppose (hypothetically) that 10 people are using the Facebook app. Five of these people report a problem, and there are eight possible events tracked in the app (</span><i><span style="font-weight: 400;">a</span></i><span style="font-weight: 400;">, </span><i><span style="font-weight: 400;">b</span></i><span style="font-weight: 400;">, …</span> <span style="font-weight: 400;">through </span><i><span style="font-weight: 400;">h</span></i><span style="font-weight: 400;">). We have 10 traces in total on these events, five in the test group (</span><i><span style="font-weight: 400;">T</span></i><span style="font-weight: 400;">) from people who encountered the bug, and five in the control group (</span><i><span style="font-weight: 400;">C</span></i><span style="font-weight: 400;">) from the remaining people who did not.</span></p>
|
||
<p><span style="font-weight: 400;">When Minesweeper is applied to these traces, it begins by extracting sequential patterns in </span><i><span style="font-weight: 400;">T</span></i><span style="font-weight: 400;"> and </span><i><span style="font-weight: 400;">C</span></i><span style="font-weight: 400;">. A sequential pattern is simply a chronological sequence of events that happened, but not necessarily one after another (i.e., there might be other events in between that were not significant).</span></p>
|
||
<p><span style="font-weight: 400;">We could visualize the data like this:</span></p>
|
||
<p><span style="font-weight: 400;">Events: { </span><i><span style="font-weight: 400;">a, b, …, h </span></i><span style="font-weight: 400;">}</span></p>
|
||
<table>
|
||
<tbody>
|
||
<tr>
|
||
<td><b>Test group</b> <i><span style="font-weight: 400;">T</span></i></td>
|
||
<td><b> Control group</b> <i><span style="font-weight: 400;">C</span></i></td>
|
||
</tr>
|
||
<tr>
|
||
<td><i><span style="font-weight: 400;">t</span></i><span style="font-weight: 400;"><sub>1</sub></span><span style="font-weight: 400;">: </span><i><span style="font-weight: 400;">a </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> b </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> c </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> d</span></i></td>
|
||
<td><i><span style="font-weight: 400;"> t</span></i><span style="font-weight: 400;"><sub>6</sub></span><span style="font-weight: 400;">: </span><i><span style="font-weight: 400;">a </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> b </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> d</span></i></td>
|
||
</tr>
|
||
<tr>
|
||
<td><i><span style="font-weight: 400;">t</span></i><span style="font-weight: 400;"><sub>2</sub></span><span style="font-weight: 400;">: </span><i><span style="font-weight: 400;">a </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> b </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> c</span></i></td>
|
||
<td><i><span style="font-weight: 400;"> t</span></i><span style="font-weight: 400;"><sub>7</sub></span><span style="font-weight: 400;">: </span><i><span style="font-weight: 400;">a </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> c </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> d</span></i></td>
|
||
</tr>
|
||
<tr>
|
||
<td><i><span style="font-weight: 400;">t</span></i><span style="font-weight: 400;"><sub>3</sub></span><span style="font-weight: 400;">: </span><i><span style="font-weight: 400;">b </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> c</span></i></td>
|
||
<td><i><span style="font-weight: 400;"> t</span></i><span style="font-weight: 400;"><sub>8</sub></span><span style="font-weight: 400;">: </span><i><span style="font-weight: 400;">a </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> c</span></i></td>
|
||
</tr>
|
||
<tr>
|
||
<td><i><span style="font-weight: 400;">t</span></i><span style="font-weight: 400;"><sub>4</sub></span><span style="font-weight: 400;">: </span><i><span style="font-weight: 400;">e </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> f </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> g </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> h</span></i></td>
|
||
<td><i><span style="font-weight: 400;"> t</span></i><span style="font-weight: 400;"><sub>9</sub></span><span style="font-weight: 400;">: </span><i><span style="font-weight: 400;">f </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> g</span></i></td>
|
||
</tr>
|
||
<tr>
|
||
<td><i><span style="font-weight: 400;">t</span></i><span style="font-weight: 400;"><sub>5</sub></span><span style="font-weight: 400;">: </span><i><span style="font-weight: 400;">e </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> g</span></i></td>
|
||
<td><i><span style="font-weight: 400;"> t</span></i><span style="font-weight: 400;"><sub>10</sub></span><span style="font-weight: 400;">: </span><i><span style="font-weight: 400;">e </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> f </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> h</span></i></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p> </p>
|
||
<p><span style="font-weight: 400;">An example of a pattern that the system can extract here is </span><i><span style="font-weight: 400;">a </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> c</span></i><span style="font-weight: 400;">. For each pattern, it computes the number of traces in which it appears in each group (</span><i><span style="font-weight: 400;">T</span></i><span style="font-weight: 400;"> and </span><i><span style="font-weight: 400;">C</span></i><span style="font-weight: 400;">). In the example above, this pattern appears twice in </span><i><span style="font-weight: 400;">T</span></i><span style="font-weight: 400;"> (</span><i><span style="font-weight: 400;">t</span></i><span style="font-weight: 400;"><sub>1</sub></span><span style="font-weight: 400;">and </span><i><span style="font-weight: 400;">t</span></i><span style="font-weight: 400;"><sub>2</sub></span><span style="font-weight: 400;">), and so its support in </span><i><span style="font-weight: 400;">T</span></i><span style="font-weight: 400;"> is 2. In this case, its support in </span><i><span style="font-weight: 400;">C</span></i><span style="font-weight: 400;"> is also 2 (</span><i><span style="font-weight: 400;">t</span></i><span style="font-weight: 400;"><sub>7 </sub></span><span style="font-weight: 400;">and </span><i><span style="font-weight: 400;">t</span></i><span style="font-weight: 400;"><sub>8</sub></span><span style="font-weight: 400;">). Note that the pattern space is combinatorial in nature, and therefore it is crucial to employ algorithms that can search this space efficiently without an exponential blowup.</span></p>
|
||
<p><span style="font-weight: 400;">Once all patterns are extracted along with their supports in </span><i><span style="font-weight: 400;">T</span></i><span style="font-weight: 400;"> and </span><i><span style="font-weight: 400;">C</span></i><span style="font-weight: 400;">, the system performs statistical isolation. For each pattern, </span><i><span style="font-weight: 400;">P</span></i><span style="font-weight: 400;">, it computes precision and recall using its support:</span></p>
|
||
<p><img decoding="async" class="alignnone wp-image-17184 size-medium" src="https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_4-e1610748472874.png?w=800" alt="" width="800" height="211" srcset="https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_4-e1610748472874.png 800w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_4-e1610748472874.png?resize=768,203 768w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_4-e1610748472874.png?resize=96,25 96w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_4-e1610748472874.png?resize=192,51 192w" sizes="(max-width: 992px) 100vw, 62vw" /></p>
|
||
<p><span style="font-weight: 400;">Informally, precision describes how accurate </span><i><span style="font-weight: 400;">P</span></i><span style="font-weight: 400;"> is in detecting whether a given trace is in the test group rather than the control group, and recall describes how much of the test group </span><i><span style="font-weight: 400;">P</span></i><span style="font-weight: 400;"> can cover. For example, the pattern </span><i><span style="font-weight: 400;">b</span></i><span style="font-weight: 400;"> has a precision of 0.75 because it occurs in four traces in total, three of which are in the test group (i.e., it is 75 percent specific to the test group). It also has a recall of 0.6 because it occurs in three out of five traces in the test group (i.e., it covers 60 percent of the test group).</span></p>
|
||
<p><span style="font-weight: 400;">The harmonic mean of the two, 0.67, is the F1-score. The system computes the F1-score of all patterns in this manner and returns the list of all patterns ranked by F1-score, as shown in the table below.</span></p>
|
||
<p><img loading="lazy" decoding="async" class="alignnone wp-image-17185 size-medium" src="https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_5.png?w=916" alt="" width="916" height="336" srcset="https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_5.png 1198w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_5.png?resize=916,336 916w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_5.png?resize=768,282 768w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_5.png?resize=1024,376 1024w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_5.png?resize=96,35 96w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_5.png?resize=192,71 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
|
||
<p><span style="font-weight: 400;">In this example, the result shows that the pattern </span><i><span style="font-weight: 400;">b </span></i><span style="font-weight: 400;">→</span><i><span style="font-weight: 400;"> c</span></i><span style="font-weight: 400;"> is the highest ranked. In a real setting, an engineer debugging the reports can infer that events </span><i><span style="font-weight: 400;">b</span></i> <span style="font-weight: 400;">and </span><i><span style="font-weight: 400;">c</span></i><span style="font-weight: 400;">, occurring in that order, are suspicious and deserve close inspection.</span></p>
|
||
<h2><span style="font-weight: 400;">Automating RCA at scale</span></h2>
|
||
<p><span style="font-weight: 400;">To make Minesweeper work at Facebook’s scale and complexity, we borrowed an idea from the data mining community —</span><a href="https://en.wikipedia.org/wiki/Sequential_pattern_mining"> <span style="font-weight: 400;">sequential pattern mining</span></a><span style="font-weight: 400;"> — to help track the order that events appear in traces.</span></p>
|
||
<p><span style="font-weight: 400;">At Facebook, it’s not unusual to encounter tens of thousands of traces related to a bug. To handle this, we leveraged the</span><a href="https://ieeexplore.ieee.org/abstract/document/914830"> <span style="font-weight: 400;">PrefixSpan</span></a><span style="font-weight: 400;"> algorithm, which is well known for being highly efficient at sequential pattern mining. To rank patterns according to their distinctiveness to the test group, thereby being useful for RCA, we utilized the aforementioned statistical approach based on the precision and recall of patterns. Finally, to make Minesweeper practical from a usability standpoint, we addressed several human-centric challenges, such as avoiding “redundant” patterns that are similar in explaining the root cause.</span></p>
|
||
<p><span style="font-weight: 400;">The figure below gives a high-level overview of the automated RCA architecture. </span></p>
|
||
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-17182" src="https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_2.png" alt="" width="1121" height="486" srcset="https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_2.png 1121w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_2.png?resize=916,397 916w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_2.png?resize=768,333 768w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_2.png?resize=1024,444 1024w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_2.png?resize=96,42 96w, https://engineering.fb.com/wp-content/uploads/2021/01/ScalableRCA_2.png?resize=192,83 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
|
||
<h2><span style="font-weight: 400;">The impact of Minesweeper</span></h2>
|
||
<p><span style="font-weight: 400;">Minesweeper has played an important role in helping Facebook’s engineers analyze and diagnose <a href="https://engineering.fb.com/2020/03/05/developer-tools/incident-tracker/">regressions</a> — sudden spikes in a group of crash or bug reports — providing insights in minutes that once could have taken days to gather. With the complexity of Facebook’s apps and the product release cycle, it is common for multiple regressions to take place simultaneously, especially after a new version is released. But thanks to Minesweeper, engineers working on user-facing regressions can analyze them promptly, making it easier than ever for them to reduce and mitigate disruptions to Facebook’s services. </span></p>
|
||
<p><span style="font-weight: 400;">More technical information is available in our paper “</span><a href="https://arxiv.org/abs/2010.09974"><span style="font-weight: 400;">Scalable statistical root cause analysis on app telemetry</span></a><span style="font-weight: 400;">.”</span></p>
|
||
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-facebook"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-facebook-17180"
|
||
class="share-facebook sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/?share=facebook"
|
||
target="_blank"
|
||
aria-labelledby="sharing-facebook-17180"
|
||
>
|
||
<span id="sharing-facebook-17180" hidden>Share on Facebook (Opens in new window)</span>
|
||
<span>Facebook</span>
|
||
</a></li><li class="share-threads"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-threads-17180"
|
||
class="share-threads sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/?share=threads"
|
||
target="_blank"
|
||
aria-labelledby="sharing-threads-17180"
|
||
>
|
||
<span id="sharing-threads-17180" hidden>Share on Threads (Opens in new window)</span>
|
||
<span>Threads</span>
|
||
</a></li><li class="share-jetpack-whatsapp"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-whatsapp-17180"
|
||
class="share-jetpack-whatsapp sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/?share=jetpack-whatsapp"
|
||
target="_blank"
|
||
aria-labelledby="sharing-whatsapp-17180"
|
||
>
|
||
<span id="sharing-whatsapp-17180" hidden>Share on WhatsApp (Opens in new window)</span>
|
||
<span>WhatsApp</span>
|
||
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-linkedin-17180"
|
||
class="share-linkedin sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/?share=linkedin"
|
||
target="_blank"
|
||
aria-labelledby="sharing-linkedin-17180"
|
||
>
|
||
<span id="sharing-linkedin-17180" hidden>Share on LinkedIn (Opens in new window)</span>
|
||
<span>LinkedIn</span>
|
||
</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-reddit-17180"
|
||
class="share-reddit sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/?share=reddit"
|
||
target="_blank"
|
||
aria-labelledby="sharing-reddit-17180"
|
||
>
|
||
<span id="sharing-reddit-17180" hidden>Share on Reddit (Opens in new window)</span>
|
||
<span>Reddit</span>
|
||
</a></li><li class="share-x"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-x-17180"
|
||
class="share-x sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/?share=x"
|
||
target="_blank"
|
||
aria-labelledby="sharing-x-17180"
|
||
>
|
||
<span id="sharing-x-17180" hidden>Share on X (Opens in new window)</span>
|
||
<span>X</span>
|
||
</a></li><li class="share-bluesky"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-bluesky-17180"
|
||
class="share-bluesky sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/?share=bluesky"
|
||
target="_blank"
|
||
aria-labelledby="sharing-bluesky-17180"
|
||
>
|
||
<span id="sharing-bluesky-17180" hidden>Share on Bluesky (Opens in new window)</span>
|
||
<span>Bluesky</span>
|
||
</a></li><li class="share-mastodon"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-mastodon-17180"
|
||
class="share-mastodon sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/?share=mastodon"
|
||
target="_blank"
|
||
aria-labelledby="sharing-mastodon-17180"
|
||
>
|
||
<span id="sharing-mastodon-17180" hidden>Share on Mastodon (Opens in new window)</span>
|
||
<span>Mastodon</span>
|
||
</a></li><li class="share-custom share-custom-hackernews"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-custom-17180"
|
||
class="share-custom share-custom-hackernews sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/?share=custom-1699562127"
|
||
target="_blank"
|
||
aria-labelledby="sharing-custom-17180"
|
||
>
|
||
<span id="sharing-custom-17180" hidden>Share on Hacker News (Opens in new window)</span>
|
||
<span style="background-image:url("https://news.ycombinator.com/y18.svg");">Hacker News</span>
|
||
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-email-17180"
|
||
class="share-email sd-button share-icon"
|
||
href="mailto:?subject=%5BShared%20Post%5D%20Minesweeper%20automates%20root%20cause%20analysis%20as%20a%20first-line%20defense%20against%20bugs&body=https%3A%2F%2Fengineering.fb.com%2F2021%2F02%2F09%2Fdeveloper-tools%2Fminesweeper%2F&share=email"
|
||
target="_blank"
|
||
aria-labelledby="sharing-email-17180"
|
||
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you're having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="05d6f35233" data-email-share-track-url="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/?share=email">
|
||
<span id="sharing-email-17180" hidden>Email a link to a friend (Opens in new window)</span>
|
||
<span>Email</span>
|
||
</a></li><li class="share-end"></li></ul></div></div></div>
|
||
|
||
</div><!-- .entry-content -->
|
||
|
||
|
||
</article><!-- #post-## -->
|
||
|
||
<footer class="entry-footer">
|
||
|
||
|
||
</footer><!-- .entry-footer -->
|
||
|
||
<div class="fb-like"
|
||
data-href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/"
|
||
data-width=""
|
||
data-layout="standard"
|
||
data-action="like"
|
||
data-size="small"
|
||
data-share="true">
|
||
</div>
|
||
|
||
|
||
<div class="related-articles">
|
||
<div class="related-articles__header">
|
||
<h3 class="related-articles__header__headline">Read More in DevInfra</h3>
|
||
<a
|
||
aria-label="View all articles in this DevInfra"
|
||
class="related-articles__header__link"
|
||
href="https://engineering.fb.com/category/developer-tools/"
|
||
>
|
||
View All </a>
|
||
</div>
|
||
|
||
<div class="article-grids related-articles__grid">
|
||
<div class="row">
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17180 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools category-networking-traffic category-production-engineering category-security category-uncategorized" id="post-24394">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/Meta-MTIA-300-Hero-final.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Meta MTIA 300 - Hero - final" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-08-24T10:45:52-07:00">
|
||
AUG 24, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/mtia-300-meta-training-chip-built-in-nics/" rel="bookmark" aria-label="Read More about MTIA 300: Meta’s First Training Chip with Built-in NICs and Communication-Offloading Engines">
|
||
MTIA 300: Meta’s First Training Chip with Built-in NICs and Communication-Offloading Engines </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17180 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools category-networking-traffic category-production-engineering category-security category-uncategorized" id="post-24128">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/10-Years-Meta-Python-HERO-large.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-06-30T09:00:46-07:00">
|
||
JUN 30, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/06/30/open-source/10-years-of-metas-commitment-to-python/" rel="bookmark" aria-label="Read More about 10 Years of Meta’s Commitment to Python">
|
||
10 Years of Meta’s Commitment to Python </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17180 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools category-networking-traffic category-production-engineering category-security category-uncategorized" id="post-23898">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/capacity_efficiency_hero_white_option_5_1775676974.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="capacity_efficiency_hero_white_option_5_1775676974" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-04-16T09:00:34-07:00">
|
||
APR 16, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/04/16/developer-tools/capacity-efficiency-at-meta-how-unified-ai-agents-optimize-performance-at-hyperscale/" rel="bookmark" aria-label="Read More about Capacity Efficiency at Meta: How Unified AI Agents Optimize Performance at Hyperscale">
|
||
Capacity Efficiency at Meta: How Unified AI Agents Optimize Performance at Hyperscale </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17180 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools category-networking-traffic category-production-engineering category-security category-uncategorized" id="post-23856">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Escaping-the-Fork-Hero-2.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Escaping the Fork - Hero 2" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-04-09T09:00:34-07:00">
|
||
APR 9, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/04/09/developer-tools/escaping-the-fork-how-meta-modernized-webrtc-across-50-use-cases/" rel="bookmark" aria-label="Read More about Escaping the Fork: How Meta Modernized WebRTC Across 50+ Use Cases">
|
||
Escaping the Fork: How Meta Modernized WebRTC Across 50+ Use Cases </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17180 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools category-networking-traffic category-production-engineering category-security category-uncategorized" id="post-23871">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Meta-Tech-Podcast-episode-84-Configuration-Safety-at-Scale.webp?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-04-08T11:25:26-07:00">
|
||
APR 8, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/04/08/security/trust-but-canary-configuration-safety-at-scale-meta-tech-podcast/" rel="bookmark" aria-label="Read More about Trust But Canary: Configuration Safety at Scale">
|
||
Trust But Canary: Configuration Safety at Scale </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17180 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools category-networking-traffic category-production-engineering category-security category-uncategorized" id="post-23840">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Compass-Not-Enclycopedia-Hero.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-04-06T09:00:50-07:00">
|
||
APR 6, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/04/06/developer-tools/how-meta-used-ai-to-map-tribal-knowledge-in-large-scale-data-pipelines/" rel="bookmark" aria-label="Read More about How Meta Used AI to Map Tribal Knowledge in Large-Scale Data Pipelines">
|
||
How Meta Used AI to Map Tribal Knowledge in Large-Scale Data Pipelines </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
</div><!-- /.row -->
|
||
</div><!--/.article-grids -->
|
||
|
||
</div><!-- /.related-articles -->
|
||
|
||
|
||
<!-- include the carousel nav here. -->
|
||
|
||
|
||
|
||
|
||
</main><!-- #main -->
|
||
|
||
</div><!-- #primary -->
|
||
|
||
<!-- Do the right sidebar check. -->
|
||
|
||
|
||
|
||
|
||
<div class="col-md-4 widget-area" id="right-sidebar" role="complementary">
|
||
<aside id="fb_related_post_widget-4" class="widget widget_fb_related_post_widget"><h3 class="widget-title">Related Posts</h3><hr>
|
||
<div class="aside-related-projects"><div class="article-grid-redesign">
|
||
<article class="post-17009 post type-post status-publish has-post-thumbnail hentry" id="post-17009">
|
||
<a href="https://engineering.fb.com/2020/12/09/data-center-engineering/how-facebook-keeps-its-large-scale-infrastructure-hardware-up-and-running/" rel="bookmark" aria-label="Read More about How Facebook keeps its large-scale infrastructure hardware up and running">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2020/12/Prineville_Server-room-dark2-e1607617109406.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2020/12/Prineville_Server-room-dark2-e1607617109406.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2020/12/Prineville_Server-room-dark2-e1607617109406.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Dec 09, 2020">Dec 09, 2020</time></span>
|
||
<h4 class="entry-title">How Facebook keeps its large-scale infrastructure hardware up and running</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div><div class="article-grid-redesign">
|
||
<article class="post-13709 post type-post status-publish has-post-thumbnail hentry" id="post-13709">
|
||
<a href="https://engineering.fb.com/2019/11/08/developer-tools/fast-dimensional-analysis/" rel="bookmark" aria-label="Read More about Fast dimensional analysis for root cause analysis at scale">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2019/11/ENG_blog_RiB_violet.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2019/11/ENG_blog_RiB_violet.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2019/11/ENG_blog_RiB_violet.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Nov 08, 2019">Nov 08, 2019</time></span>
|
||
<h4 class="entry-title">Fast dimensional analysis for root cause analysis at scale</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div><div class="article-grid-redesign">
|
||
<article class="post-10705 post type-post status-publish has-post-thumbnail hentry" id="post-10705">
|
||
<a href="https://engineering.fb.com/2018/11/06/developer-tools/getafix-how-facebook-tools-learn-to-fix-bugs-automatically/" rel="bookmark" aria-label="Read More about Getafix: How Facebook tools learn to fix bugs automatically">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2018/11/Getafix_animation_7.gif?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2018/11/Getafix_animation_7.gif?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2018/11/Getafix_animation_7.gif?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Nov 06, 2018">Nov 06, 2018</time></span>
|
||
<h4 class="entry-title">Getafix: How Facebook tools learn to fix bugs automatically</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div></div></aside><aside id="wpb_widget-3" class="widget widget_wpb_widget"><h3 class="widget-title">Related Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2316600655751692/>
|
||
<div class="job-description">
|
||
Machine Learning Engineer
|
||
</div>
|
||
<div class="job-location">
|
||
TEL AVIV, IL
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2887906508264541/>
|
||
<div class="job-description">
|
||
Software Engineer (Leadership) - Product
|
||
</div>
|
||
<div class="job-location">
|
||
LONDON, GB
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
SUNNYVALE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
BELLEVUE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
MENLO PARK, US
|
||
</div>
|
||
</a>
|
||
</li> </ul>
|
||
<a href="https://www.metacareers.com">
|
||
<div class="btn available-positions-btn">
|
||
See All Jobs </div>
|
||
</a>
|
||
</aside>
|
||
</div><!-- #secondary -->
|
||
|
||
|
||
|
||
<!-- <php get_template_part('content-single-side-bar'); ?> -->
|
||
|
||
<!-- Do the right sidebar check. -->
|
||
|
||
</div><!-- Container end -->
|
||
|
||
</div><!-- Wrapper end -->
|
||
|
||
</div>
|
||
|
||
<!-- ******************* The Footer Full-width Widget Area ******************* -->
|
||
|
||
<div class="wrapper wrapper-community-outer">
|
||
<div class="wrapper-community">
|
||
|
||
<div class="community" tabindex="-1">
|
||
|
||
<div class="row">
|
||
<!-- community section is generated through WordPress widget -->
|
||
<div id="wpb_widget-2" class="footer-widget widget_wpb_widget widget-count-3 col-md-4"><h3 class="widget-title">Available Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2316600655751692/>
|
||
<div class="job-description">
|
||
Machine Learning Engineer
|
||
</div>
|
||
<div class="job-location">
|
||
TEL AVIV, IL
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2887906508264541/>
|
||
<div class="job-description">
|
||
Software Engineer (Leadership) - Product
|
||
</div>
|
||
<div class="job-location">
|
||
LONDON, GB
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
SUNNYVALE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
BELLEVUE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
MENLO PARK, US
|
||
</div>
|
||
</a>
|
||
</li> </ul>
|
||
<a href="https://www.metacareers.com">
|
||
<div class="btn available-positions-btn">
|
||
See All Jobs </div>
|
||
</a>
|
||
</div><!-- .footer-widget --><div id="custom_html-2" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Technology at Meta</h3><div class="textwidget custom-html-widget"><ul class="stay-connected-lists">
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-fb-engineering" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Engineering at Meta - X
|
||
</div>
|
||
<a onclick='return twitterFollow("Meta_Engineers")'>
|
||
<button class="btn stay-connected-btn">
|
||
Follow
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-AI" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
AI at Meta
|
||
</div>
|
||
<a href="https://ai.meta.com/blog/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta Quest Blog
|
||
</div>
|
||
<a href="https://www.meta.com/blog/quest/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta for Developers
|
||
</div>
|
||
<a href="https://developers.facebook.com/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-bug-bounty" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta Bug Bounty
|
||
</div>
|
||
<a href="https://bugbounty.meta.com/">
|
||
<button class="btn stay-connected-btn">
|
||
Learn more
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
|
||
<li class="stay-connected-item">
|
||
<img class="stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/rss.png" alt="footer-rss" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
RSS
|
||
</div>
|
||
<a href="https://code.facebook.com/posts/rss/">
|
||
<button class="btn stay-connected-btn">
|
||
Subscribe
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div></div><!-- .footer-widget --><div id="custom_html-3" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Open Source</h3><div class="textwidget custom-html-widget"><p class="opensource-desc">
|
||
Meta believes in building community through open source technology. Explore our latest projects in Artificial Intelligence, Data Infrastructure, Development Tools, Front End, Languages, Platforms, Security, Virtual Reality, and more.
|
||
</p>
|
||
<ul class="opensource-cats">
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/android.png" alt="android" loading="lazy" width="50" height="59">
|
||
<div class="opensource-cap">
|
||
ANDROID
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/ios.png" alt="ios" loading="lazy" width="48" height="58">
|
||
<div class="opensource-cap">
|
||
iOS
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/web.png" alt="web" loading="lazy" width="55" height="49">
|
||
<div class="opensource-cap">
|
||
WEB
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/backend.png" alt="backend" loading="lazy" width="56" height="51">
|
||
<div class="opensource-cap">
|
||
BACKEND
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/hardware.png" alt="hardware" loading="lazy" width="55" height="44">
|
||
<div class="opensource-cap">
|
||
HARDWARE
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
|
||
<button class="btn opensource-btn" onclick="window.location.href='https://opensource.fb.com/'">
|
||
Learn More
|
||
</button>
|
||
|
||
</div></div><!-- .footer-widget -->
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</div><!-- #wrapper-footer-full -->
|
||
|
||
|
||
<div class="wrapper-footer" id="wrapper-footer">
|
||
<div class="inner-container">
|
||
<div class="footer">
|
||
<div class="row">
|
||
<div class="col-md-12">
|
||
<footer class="meta-footer" id="meta-footer">
|
||
<div class="site-info row justify-content-between">
|
||
<div class="col-md-5 meta-logo-container">
|
||
<a href="https://about.facebook.com/">
|
||
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/meta_logo_full.svg" width="89" height="18" alt="Meta" />
|
||
</a>
|
||
<div id="text-2" class="widget widget_text"> <div class="textwidget"><p>Engineering at Meta is a technical news resource for engineers interested in how we solve large-scale technical challenges at Meta.</p>
|
||
</div>
|
||
</div> </div>
|
||
<div class="col-md-6 footer-meta-links-container">
|
||
<ul class="meta-footer-links">
|
||
<li class="footer-link"><a href="https://engineering.fb.com">Home</a></li><li class="footer-link"><a href="https://about.meta.com/">Company Info</a></li><li class="footer-link"><a href="https://www.metacareers.com/?ref=engineering.fb.com">Careers</a></li> </ul>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="footer">
|
||
<div class="row">
|
||
|
||
<div class="col-md-12">
|
||
|
||
<footer class="site-footer" id="colophon">
|
||
<div class="inner-container">
|
||
<div class="site-info row justify-content-between">
|
||
|
||
<div class="col-md-4 copyright-container">
|
||
<div class="copyright"> © 2026 Meta</div>
|
||
</div>
|
||
|
||
<!-- static footer -->
|
||
<div class="col-md-7 footer-links-container">
|
||
<ul class="footer-links">
|
||
<li class="footer-link"><a href="https://www.facebook.com/policies">Terms</a></li><li class="footer-link"><a href="https://www.facebook.com/privacy/policy">Privacy</a></li><li class="footer-link"><a href="/privacy">Cookies</a></li><li class="footer-link"><a href="https://www.facebook.com/help">Help</a></li> </ul>
|
||
</div>
|
||
</div>
|
||
</div><!-- .site-info -->
|
||
|
||
</footer><!-- #colophon -->
|
||
|
||
</div><!--col end -->
|
||
|
||
</div><!-- row end -->
|
||
|
||
</div><!-- //.inner-container -->
|
||
</div><!-- wrapper end -->
|
||
|
||
</div><!-- #page we need this extra closing tag here -->
|
||
|
||
<script type="speculationrules">
|
||
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/code-fb-com/*","/wp-content/themes/understrap/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
|
||
</script>
|
||
<div class="gdprconsent-container" id="GDPRConsentBar">
|
||
<div class="gdprconsent-wrapper">
|
||
<div class="gdprconsent-content">
|
||
<p>To help personalize content, tailor and measure ads and provide a safer experience, we use cookies. By clicking or navigating the site, you agree to allow our collection of information on and off Facebook through cookies. Learn more, including about available controls: <a href="/privacy">Cookie Policy</a></p>
|
||
<div class="gdprconsent-button">
|
||
<button type="button" data-gdpr-accept>Accept</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Twitter universal website tag code -->
|
||
<script>
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
|
||
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
|
||
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
|
||
// Insert Twitter Pixel ID and Standard Event data below
|
||
twq('init','o31u0');
|
||
twq('track','PageView');
|
||
});
|
||
}
|
||
</script>
|
||
<!-- End Twitter universal website tag code -->
|
||
|
||
|
||
<script type="text/javascript">
|
||
window.WPCOM_sharing_counts = {"https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/":17180};
|
||
</script>
|
||
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KZRLGZ8" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
<noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=660742068067366&ev=PageView&noscript=1"/></noscript>
|
||
<script id="child-understrap-scripts-js-extra">
|
||
var loadmore_params = {"restfulURL":"https://engineering.fb.com/wp-json/fb/v1/loadmore","searchText":"","posts":"{\"page\":0,\"year\":2021,\"monthnum\":2,\"day\":9,\"name\":\"minesweeper\",\"category_name\":\"uncategorized\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","post_type":"post"};
|
||
var search_params = {"ajaxurl":"https://engineering.fb.com/wp-admin/admin-ajax.php","posts":"{\"page\":0,\"year\":2021,\"monthnum\":2,\"day\":9,\"name\":\"minesweeper\",\"category_name\":\"uncategorized\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","article_filter":"2f23ad5ce3"};
|
||
var more_articles = {"next_article":{"label":"Next","permalink":"https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/","title":"Faster, more efficient systems for finding and fixing regressions","thumbnail":["https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_HERO_FINAL.jpg?w=580&h=326&crop=1",580,326,true]},"prev_article":{"label":"Prev","permalink":"https://engineering.fb.com/2021/02/05/open-source/hsthrift/","title":"Open-sourcing Thrift for Haskell","thumbnail":["https://engineering.fb.com/wp-content/uploads/2020/10/OSiB_RichTeal.jpg?w=580&h=326&crop=1",580,326,true]}};
|
||
//# sourceURL=child-understrap-scripts-js-extra
|
||
</script>
|
||
<script type="text/javascript" src="https://engineering.fb.com/_static/??/wp-content/themes/code-fb-com/dist/js/child-theme.min.js,/wp-content/themes/code-fb-com/js/prism.js?m=1783017802j" ></script><script id="sharing-js-js-extra">
|
||
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":""};
|
||
//# sourceURL=sharing-js-js-extra
|
||
</script>
|
||
<script id="sharing-js-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/_inc/build/sharedaddy/sharing.min.js?ver=16.1.3"></script>
|
||
<script id="sharing-js-js-after">
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-facebook' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-threads' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-threads' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomthreads', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-linkedin' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-x' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-x' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomx', 'menubar=1,resizable=1,width=600,height=350' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-bluesky' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-bluesky' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcombluesky', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-mastodon' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-mastodon' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcommastodon', 'menubar=1,resizable=1,width=460,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
//# sourceURL=sharing-js-js-after
|
||
</script>
|
||
<script id="video-analytics-js-after">
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
let youtube = document.querySelectorAll('[src*="youtube-nocookie"]');
|
||
for (let i = 0; i < youtube.length; i++) {
|
||
youtube[i].src = youtube[i].src.replace('youtube-nocookie.com', 'youtube.com');
|
||
}
|
||
let vimeo = document.querySelectorAll('[src*="vimeo"]');
|
||
for (let i = 0; i < vimeo.length; i++) {
|
||
vimeo[i].src = vimeo[i].src.replace('&dnt=true', '');
|
||
}
|
||
});
|
||
}
|
||
//# sourceURL=video-analytics-js-after
|
||
</script>
|
||
<script id="wp-emoji-settings" type="application/json">
|
||
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://engineering.fb.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
|
||
</script>
|
||
<script type="module">
|
||
/*! This file is auto-generated */
|
||
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
|
||
//# sourceURL=https://engineering.fb.com/wp-includes/js/wp-emoji-loader.min.js
|
||
</script>
|
||
|
||
</body>
|
||
|
||
</html>
|