1259 lines
105 KiB
HTML
1259 lines
105 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en-US">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||
<meta name="mobile-web-app-capable" content="yes">
|
||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||
<meta name="apple-mobile-web-app-title" content="Engineering at Meta - Engineering at Meta Blog">
|
||
<meta property="fb:app_id" content="1425766027653270" />
|
||
<link rel="profile" href="http://gmpg.org/xfn/11">
|
||
<link rel="pingback" href="https://engineering.fb.com/xmlrpc.php">
|
||
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
|
||
<script async src="https://engineering.fb.com/wp-content/themes/code-fb-com/dist/js/optimize.min.js"></script>
|
||
|
||
<!-- This site is optimized with the Yoast SEO Premium plugin v19.3 (Yoast SEO v27.3) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
|
||
<title>Fix Fast for finding and fixing regressions - Engineering at Meta</title>
|
||
<meta name="description" content="The goal of Fix Fast is to reduce the effort required to fix regressions by improving engineers' experience and moving detection upstream." />
|
||
<link rel="canonical" href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/" />
|
||
<meta name="twitter:label1" content="Written by" />
|
||
<meta name="twitter:data1" content="Jian Zhang, Brian Keller, changigi649" />
|
||
<meta name="twitter:label2" content="Est. reading time" />
|
||
<meta name="twitter:data2" content="10 minutes" />
|
||
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/#article","isPartOf":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/"},"author":[{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/c7ff8a1b1258049f8705a43611684c9e"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/a798f4d0d55d7f4fa9b58f01564cad5b"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/7167c692371a46d5bb5879bbd971c6c0"}],"headline":"Faster, more efficient systems for finding and fixing regressions","datePublished":"2021-02-17T16:59:24+00:00","dateModified":"2021-03-29T04:17:07+00:00","mainEntityOfPage":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/"},"wordCount":1909,"publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"image":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/02\/FixFast_HERO_FINAL.jpg","articleSection":["DevInfra"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/","url":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/","name":"Fix Fast for finding and fixing regressions - Engineering at Meta","isPartOf":{"@id":"https:\/\/engineering.fb.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/#primaryimage"},"image":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/02\/FixFast_HERO_FINAL.jpg","datePublished":"2021-02-17T16:59:24+00:00","dateModified":"2021-03-29T04:17:07+00:00","description":"The goal of Fix Fast is to reduce the effort required to fix regressions by improving engineers' experience and moving detection upstream.","breadcrumb":{"@id":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/#primaryimage","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/02\/FixFast_HERO_FINAL.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/02\/FixFast_HERO_FINAL.jpg","width":1920,"height":1080,"caption":"Fix Fast: Faster, more efficient systems for finding and fixing regressions"},{"@type":"BreadcrumbList","@id":"https:\/\/engineering.fb.com\/2021\/02\/17\/developer-tools\/fix-fast\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/engineering.fb.com\/"},{"@type":"ListItem","position":2,"name":"Faster, more efficient systems for finding and fixing regressions"}]},{"@type":"WebSite","@id":"https:\/\/engineering.fb.com\/#website","url":"https:\/\/engineering.fb.com\/","name":"Engineering at Meta","description":"Engineering at Meta Blog","publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/engineering.fb.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/engineering.fb.com\/#organization","name":"Meta","url":"https:\/\/engineering.fb.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","width":29011,"height":12501,"caption":"Meta"},"image":{"@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/Engineering\/","https:\/\/x.com\/fb_engineering"]},[],{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/c7ff8a1b1258049f8705a43611684c9e","name":"Jian Zhang","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Jian Zhang"},"url":"https:\/\/engineering.fb.com\/author\/jian-zhang\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/a798f4d0d55d7f4fa9b58f01564cad5b","name":"Brian Keller","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Brian Keller"},"url":"https:\/\/engineering.fb.com\/author\/brian-keller\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/7167c692371a46d5bb5879bbd971c6c0","name":"changigi649","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/a090692c0af9b5ed6f8071e0a05e3147f0ba61b52a51c0671d073d9adb4e198e?s=96&d=mm&r=g8fd1e2b34e75c09c4aea578e73cbde79","url":"https:\/\/secure.gravatar.com\/avatar\/a090692c0af9b5ed6f8071e0a05e3147f0ba61b52a51c0671d073d9adb4e198e?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/a090692c0af9b5ed6f8071e0a05e3147f0ba61b52a51c0671d073d9adb4e198e?s=96&d=mm&r=g","caption":"changigi649"},"url":"https:\/\/engineering.fb.com\/author\/changigi649\/"}]}</script>
|
||
<!-- / Yoast SEO Premium plugin. -->
|
||
|
||
|
||
<link rel='dns-prefetch' href='//secure.gravatar.com' />
|
||
<link rel='dns-prefetch' href='//v0.wordpress.com' />
|
||
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta » Feed" href="https://engineering.fb.com/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta » Comments Feed" href="https://engineering.fb.com/comments/feed/" />
|
||
<script async src="https://www.googleoptimize.com/optimize.js?id=OPT-TPNDQC5" onerror="dataLayer.hide.end && dataLayer.hide.end()"></script>
|
||
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2021%2F02%2F17%2Fdeveloper-tools%2Ffix-fast%2F" />
|
||
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2021%2F02%2F17%2Fdeveloper-tools%2Ffix-fast%2F&format=xml" />
|
||
<style id="wp-img-auto-sizes-contain-inline-css">
|
||
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
|
||
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
|
||
</style>
|
||
<style id="wp-emoji-styles-inline-css">
|
||
|
||
img.wp-smiley, img.emoji {
|
||
display: inline !important;
|
||
border: none !important;
|
||
box-shadow: none !important;
|
||
height: 1em !important;
|
||
width: 1em !important;
|
||
margin: 0 0.07em !important;
|
||
vertical-align: -0.1em !important;
|
||
background: none !important;
|
||
padding: 0 !important;
|
||
}
|
||
/*# sourceURL=wp-emoji-styles-inline-css */
|
||
</style>
|
||
<style id="wp-block-library-inline-css">
|
||
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
|
||
|
||
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
|
||
</style>
|
||
<style id="classic-theme-styles-inline-css">
|
||
/*! This file is auto-generated */
|
||
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
|
||
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
|
||
</style>
|
||
|
||
<style id="global-styles-inline-css">
|
||
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
|
||
/*# sourceURL=global-styles-inline-css */
|
||
</style>
|
||
|
||
<link rel='stylesheet' id='all-css-12' href='https://engineering.fb.com/_static/??-eJyNjUsOwjAMRC9EaiqghQXiKFVqhzSQn+pEiNuTFJBAbNjZTzNv4BYFBp+UTxBt1sYzFKQpzpVz5WQ4QSUv0CDzCj6KaVJOMWAgJc5jwe7ZKTnAyVgSS6Jxxv90XRbv3YtKUeJVtF3TgguUbZHyJGdFkui+nMbrvxWD8QhjLvvAAY20wgYd+OupspM7tv3+sO533Wb7AImQY4U=' type='text/css' media='all' />
|
||
<script id="gdprconsent-js" src="https://engineering.fb.com/wp-content/plugins/wp-gdpr-consent/dist/gdprconsent.js?ver=2.0.0"></script>
|
||
<script id="gdprconsent-js-after">
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack('https://www.googletagmanager.com/gtag/js?id=391063774');
|
||
window.gdprSafeTrack(function() {
|
||
var google_analytics_g4 = decodeURIComponent( '391063774' );
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
gtag('js', new Date());
|
||
gtag('config', google_analytics_g4);
|
||
});
|
||
}
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
var google_tag_manager = decodeURIComponent( 'GTM-KZRLGZ8' );
|
||
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||
})(window,document,'script','dataLayer',google_tag_manager);
|
||
});
|
||
}
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
var facebook_pixel = decodeURIComponent( '660742068067366' );
|
||
!function(f,b,e,v,n,t,s)
|
||
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
|
||
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
|
||
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
|
||
n.queue=[];t=b.createElement(e);t.async=!0;
|
||
t.src=v;s=b.getElementsByTagName(e)[0];
|
||
s.parentNode.insertBefore(t,s)}(window, document,'script',
|
||
'https://connect.facebook.net/en_US/fbevents.js');
|
||
fbq('init', facebook_pixel);
|
||
fbq('track', 'PageView');
|
||
});
|
||
}
|
||
//# sourceURL=gdprconsent-js-after
|
||
</script>
|
||
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJzTLy/QzcxLzilNSS3WzwKiwtLUokoopZebmaeXVayjj0+Rbm5melFiSSpUsX2uraG5hbmZgYmloUkWAK+3Iic=" ></script><script type="text/javascript" src="https://engineering.fb.com/wp-content/themes/understrap/js/popper.min.js?m=1773259670g" ></script><link rel="https://api.w.org/" href="https://engineering.fb.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.fb.com/wp-json/wp/v2/posts/17271" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.fb.com/xmlrpc.php?rsd" />
|
||
<meta name="generator" content="WordPress 7.1" />
|
||
<link rel='shortlink' href='https://wp.me/pa0Lhq-4uz' />
|
||
<link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/Optimistic_Display_W_Md.woff2" as="font" type="font/woff2" crossorigin="anonymous">
|
||
<!-- Jetpack Open Graph Tags -->
|
||
<meta property="og:type" content="article" />
|
||
<meta property="og:title" content="Faster, more efficient systems for finding and fixing regressions" />
|
||
<meta property="og:url" content="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/" />
|
||
<meta property="og:description" content="Every workday, Facebook engineers commit thousands of diffs (which is a change consisting of one or more files) into production. This code velocity allows us to rapidly ship new features, deliver b…" />
|
||
<meta property="article:published_time" content="2021-02-17T16:59:24+00:00" />
|
||
<meta property="article:modified_time" content="2021-03-29T04:17:07+00:00" />
|
||
<meta property="og:site_name" content="Engineering at Meta" />
|
||
<meta property="og:image" content="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_HERO_FINAL.jpg" />
|
||
<meta property="og:image:width" content="1920" />
|
||
<meta property="og:image:height" content="1080" />
|
||
<meta property="og:image:alt" content="Fix Fast: Faster, more efficient systems for finding and fixing regressions" />
|
||
<meta property="og:locale" content="en_US" />
|
||
<meta name="twitter:text:title" content="Faster, more efficient systems for finding and fixing regressions" />
|
||
<meta name="twitter:image" content="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_HERO_FINAL.jpg?w=640" />
|
||
<meta name="twitter:image:alt" content="Fix Fast: Faster, more efficient systems for finding and fixing regressions" />
|
||
<meta name="twitter:card" content="summary_large_image" />
|
||
|
||
<!-- End Jetpack Open Graph Tags -->
|
||
<style id="wp-custom-css">
|
||
.table-of-contents {
|
||
position: sticky;
|
||
top: 20px;
|
||
background: #fff;
|
||
padding: 1em;
|
||
border: 1px solid #ddd;
|
||
border-radius: 8px;
|
||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||
max-width: 250px;
|
||
}
|
||
</style>
|
||
|
||
<link rel="shortcut icon" href="https://engineering.fb.com/wp-content/themes/code-fb-com/favicon.ico" />
|
||
</head>
|
||
|
||
<body class="wp-singular post-template-default single single-post postid-17271 single-format-standard wp-theme-understrap wp-child-theme-code-fb-com group-blog">
|
||
|
||
<div class="hfeed site" id="page">
|
||
|
||
<!-- ******************* The Navbar Area ******************* -->
|
||
<div class="wrapper-fluid wrapper-navbar" id="wrapper-navbar">
|
||
<a class="skip-link screen-reader-text sr-only" href="#content">
|
||
Skip to content </a>
|
||
|
||
<nav class="navbar navbar-expand-std">
|
||
|
||
<div class="nav-container">
|
||
|
||
|
||
<!-- Your site title as branding in the menu -->
|
||
|
||
<div class="nav-top-row">
|
||
|
||
<h1 class="navbar-brand mb-0">
|
||
<a rel="home" href="https://engineering.fb.com/" title="Engineering at Meta">
|
||
<img class="navbar-brand-logo" src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/logo-meta.svg" class="navbar-brand-logo" alt="Engineering at Meta" width="215" height="40" loading="lazy">
|
||
</a>
|
||
</h1>
|
||
|
||
<!-- end custom logo -->
|
||
<label for="search" class="screen-reader-text">Search this site</label>
|
||
<input type="text" id="search" name="search" class="form-control-lg navbar-search" autocomplete="off">
|
||
|
||
<button class="navbar-toggler navbar-searchbar-toggler"
|
||
type="button"
|
||
data-toggle="collapse"
|
||
data-target="#searchbarDropdown"
|
||
aria-controls="searchbarDropdown"
|
||
aria-expanded="false"
|
||
aria-label="Toggle navigation">
|
||
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/icon-search.svg" class="navbar-mobile-search-icon" loading="lazy" width="24" height="25" />
|
||
</button>
|
||
|
||
<!-- desktop language menu (mobile language menu is implemented via jQuery)-->
|
||
<div class="notranslate nice-select navbar-language form-control">
|
||
</div>
|
||
|
||
<button class="navbar-toggler collapsed navbar-menu-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
|
||
</div>
|
||
|
||
<div class="navbar-border"></div>
|
||
|
||
<!-- The WordPress Menu goes here -->
|
||
<div id="navbarNavDropdown" class="collapse navbar-collapse"><ul id="main-menu" class="navbar-nav"><li id="menu-item-60" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-60 dropdown"><a title="Open Source" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Open Source <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-12330" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-12330"><a title="Open Source" href="https://engineering.fb.com/category/open-source/" class="nav-link">Open Source</a></li>
|
||
<li id="menu-item-12331" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-12331"><a title="Meta Open Source" href="https://opensource.fb.com" class="nav-link">Meta Open Source</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2686" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2686 dropdown"><a title="Platforms" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Platforms <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-65" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-65"><a title="Android" href="https://engineering.fb.com/category/android/" class="nav-link">Android</a></li>
|
||
<li id="menu-item-66" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-66"><a title="iOS" href="https://engineering.fb.com/category/ios/" class="nav-link">iOS</a></li>
|
||
<li id="menu-item-67" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-67"><a title="Web" href="https://engineering.fb.com/category/web/" class="nav-link">Web</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2682" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2682 dropdown"><a title="Infrastructure Systems" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Infrastructure Systems <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-218" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-218"><a title="Core Infra" href="https://engineering.fb.com/category/core-infra/" class="nav-link">Core Infra</a></li>
|
||
<li id="menu-item-219" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-219"><a title="Data Infrastructure" href="https://engineering.fb.com/category/data-infrastructure/" class="nav-link">Data Infrastructure</a></li>
|
||
<li id="menu-item-220" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-220"><a title="DevInfra" href="https://engineering.fb.com/category/developer-tools/" class="nav-link">DevInfra</a></li>
|
||
<li id="menu-item-226" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-226"><a title="Production Engineering" href="https://engineering.fb.com/category/production-engineering/" class="nav-link">Production Engineering</a></li>
|
||
<li id="menu-item-227" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-227"><a title="Security & Privacy" href="https://engineering.fb.com/category/security/" class="nav-link">Security & Privacy</a></li>
|
||
<li id="menu-item-18607" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18607"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/systems-infrastructure/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2683" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2683 dropdown"><a title="Physical Infrastructure" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Physical Infrastructure <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-215" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-215"><a title="Connectivity" href="https://engineering.fb.com/category/connectivity/" class="nav-link">Connectivity</a></li>
|
||
<li id="menu-item-216" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-216"><a title="Data Center Engineering" href="https://engineering.fb.com/category/data-center-engineering/" class="nav-link">Data Center Engineering</a></li>
|
||
<li id="menu-item-217" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-217"><a title="Networking & Traffic" href="https://engineering.fb.com/category/networking-traffic/" class="nav-link">Networking & Traffic</a></li>
|
||
<li id="menu-item-18608" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18608"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/networking-connectivity/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2684" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2684 dropdown"><a title="Video Engineering & AR/VR" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Video Engineering & AR/VR <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-225" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-225"><a title="Video Engineering" href="https://engineering.fb.com/category/video-engineering/" class="nav-link">Video Engineering</a></li>
|
||
<li id="menu-item-223" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-223"><a title="Virtual Reality" href="https://engineering.fb.com/category/virtual-reality/" class="nav-link">Virtual Reality</a></li>
|
||
<li id="menu-item-13388" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13388"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/augmented-reality-virtual-reality/" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-2685" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2685 dropdown"><a title="Artificial Intelligence" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Artificial Intelligence <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
|
||
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
|
||
</svg></a>
|
||
<ul class=" dropdown-menu" role="menu">
|
||
<li id="menu-item-222" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-222"><a title="ML Applications" href="https://engineering.fb.com/category/ml-applications/" class="nav-link">ML Applications</a></li>
|
||
<li id="menu-item-221" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-221"><a title="AI Research" href="https://engineering.fb.com/category/ai-research/" class="nav-link">AI Research</a></li>
|
||
<li id="menu-item-13387" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13387"><a title="Research Publications" href="https://ai.facebook.com/results/?content_types%5B0%5D=publication" class="nav-link">Research Publications</a></li>
|
||
</ul>
|
||
</li>
|
||
<li id="menu-item-228" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-228"><a title="Watch Videos" href="/videos" class="nav-link">Watch Videos</a></li>
|
||
</ul></div> </div><!-- .container -->
|
||
|
||
<div class="search-container collapse" id="searchbarDropdown">
|
||
<div class="searchbar-search">
|
||
<div class="searchbar-container">
|
||
|
||
<div class="navbar-mobile-search-margin"></div>
|
||
<input type="text" name="search" placeholder="Search" class="form-control-lg navbar-mobile-search" autofocus autocomplete="off">
|
||
<button class="searchbar-clear">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</nav><!-- .site-navigation -->
|
||
|
||
</div><!-- .wrapper-navbar end -->
|
||
|
||
<div class="wrapper" id="single-wrapper">
|
||
|
||
<div class="single-container row" id="content" tabindex="-1">
|
||
|
||
<!-- Do the left sidebar check. -->
|
||
|
||
|
||
|
||
<div class="content-area" id="primary">
|
||
<main class="site-main" id="main">
|
||
|
||
|
||
|
||
<article class="post-17271 post type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-17271">
|
||
|
||
<header class="entry-header">
|
||
<div class="entry-meta">
|
||
POSTED ON <span class="posted-on entry-date"><time class="published updated" datetime="2021-02-17">FEBRUARY 17, 2021</time></span> TO <span class="cat-links"><a href="https://engineering.fb.com/category/developer-tools/" rel="category tag">DevInfra</a></span>
|
||
<h1 class="entry-title"><div>Faster, more efficient systems for finding and fixing regressions</div></h1> </div><!-- .entry-meta -->
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
<figure id="post-feat-image-container">
|
||
<img width="1920" height="1080" src="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_HERO_FINAL.jpg" class="wp-post-image" alt="Fix Fast: Faster, more efficient systems for finding and fixing regressions" title="FixFast_HERO" /> </figure>
|
||
|
||
<div class="entry-authors-social">
|
||
<div class="entry-authors">
|
||
By <a href="https://engineering.fb.com/author/jian-zhang/" title="Posts by Jian Zhang" class="author url fn" rel="author">Jian Zhang</a>, <a href="https://engineering.fb.com/author/brian-keller/" title="Posts by Brian Keller" class="author url fn" rel="author">Brian Keller</a> </div>
|
||
</div>
|
||
|
||
|
||
<div class="entry-content">
|
||
|
||
<p><span style="font-weight: 400;">Every workday, Facebook engineers commit thousands of diffs (which is a change consisting of one or more files) into production. This code velocity allows us to rapidly ship new features, deliver bug fixes and optimizations, and run experiments. However, a natural downside to moving quickly in any industry is the risk of inadvertently causing regressions in performance, reliability, or functional correctness.</span></p>
|
||
<p><span style="font-weight: 400;">To enable us to scale our developer infrastructure to this volume — while also mitigating and minimizing regressions — we build tools and mechanisms to help engineers understand whether a diff has introduced (or will introduce) a regression. We consider the feedback we get from these tools as signals. Signals</span> <span style="font-weight: 400;">can be generated by automated tests, static analysis frameworks, performance logs, crash dumps, bug reports, production monitoring alarms, and dozens of other sources. These signals can be surfaced at any stage of the development life cycle. A given diff can generate hundreds of signals — including errors, successes, and warnings — that can help an engineer assess whether that diff is ready to be landed into a stable branch and begin making its way into production. After a diff has landed, signals are usually delivered to engineers via our task management system. An individual engineer can easily have hundreds of tasks in their backlog.</span></p>
|
||
<p><span style="font-weight: 400;">With so many signals being generated, it can quickly become overwhelming for engineers to know where to focus their time. A very important signal could get overlooked because it was drowned out by noisy, lower-priority signals — making it harder for engineers to quickly or easily assess the relative priority for each signal. They might spend hours debugging an issue that turned out to be noise, or lack the necessary information to help them diagnose and fix it. </span></p>
|
||
<p><span style="font-weight: 400;">Over time, as we built more tools and monitoring systems, the signal volume increased until it became a persistent distraction. Eventually, we realized we needed a more effective way to help engineers easily see what to work on first, and spend less time debugging and fixing issues. </span></p>
|
||
<h2><span style="font-weight: 400;">Making fixes more efficient </span></h2>
|
||
<p><span style="font-weight: 400;">We’ve seen that defects become exponentially more time consuming to fix as they progress further along the development process. This is true for most manufacturing industries, whether you are producing automobiles, building skyscrapers, or shipping software. We clearly see this in our own data:</span></p>
|
||
<ul>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">A defect detected in the integrated development environment (IDE) while an engineer is coding, can be fixed in minutes.</span></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">A defect detected while a diff is being reviewed can be fixed in hours</span><b>.</b></li>
|
||
<li style="font-weight: 400;" aria-level="1"><span style="font-weight: 400;">If that same defect is allowed to land and make its way into production, it can take days to deliver a fix.</span></li>
|
||
</ul>
|
||
<figure id="attachment_17273" aria-describedby="caption-attachment-17273" style="width: 2000px" class="wp-caption alignnone"><img decoding="async" class="size-full wp-image-17273" src="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-01_FINAL.jpg" alt="Average time required to fix issues increases exponentially the further a defect makes it in the development stage. " width="2000" height="1070" srcset="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-01_FINAL.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-01_FINAL.jpg?resize=916,490 916w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-01_FINAL.jpg?resize=768,411 768w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-01_FINAL.jpg?resize=1024,548 1024w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-01_FINAL.jpg?resize=1536,822 1536w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-01_FINAL.jpg?resize=96,51 96w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-01_FINAL.jpg?resize=192,103 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-17273" class="wp-caption-text">Average time required to fix issues increases exponentially the further a defect makes it in the development stage.</figcaption></figure>
|
||
<p><span style="font-weight: 400;">Any solution designed to decrease fix times would need to ensure that regressions were being detected and fixed</span> <span style="font-weight: 400;">as early as possible in the process.</span></p>
|
||
<h2><span style="font-weight: 400;">Introducing Fix Fast</span></h2>
|
||
<p><span style="font-weight: 400;">To address these challenges, we launched our cross-functional effort, dubbed Fix Fast, in 2019. Fix Fast brings together the engineering teams responsible for generating the lion’s share of signals representing regressions in performance, reliability, and functional correctness. The mission of this effort was to reduce the effort required to fix regressions at scale by improving the engineering experience and by moving actionable detection further upstream.</span></p>
|
||
<p><span style="font-weight: 400;">While the high-level strategy was clear, we still had some big questions to answer: Which problems should we start with? How should we measure progress? We knew we’d need a data-driven approach to execute against this goal at Facebook scale, so we started by finding a way to measure the level of effort and set benchmarks.</span></p>
|
||
<h2><span style="font-weight: 400;">A metric for faster fixes</span></h2>
|
||
<p><span style="font-weight: 400;">In pursuit of a metric, we considered many options. Ultimately, we landed on a metric that would incentivize engineers to “shift left,” a common industry practice of detecting (and fixing) regressions as early in the development life cycle as possible. As mentioned above, helping engineers detect and fix an issue in the IDE while they are still focused on that block of code is significantly less time-consuming doing so once the code is live in production.</span></p>
|
||
<p><span style="font-weight: 400;">We developed a uniform top-level metric, called cost per developer (CPD), such that earlier stages of a fix were given a lower weighted cost than later stages. (Note that the term </span><i><span style="font-weight: 400;">cost</span></i><span style="font-weight: 400;"> is used to represent the amount of time an engineer spends fixing an issue, as opposed to a direct dollar amount.) Weights for each stage were established by analyzing internal data and applying rigorous backtesting to determine the relative amount of time engineers were already spending fixing regressions in each stage. This approach encouraged detection and prevention of issues in the earlier stages of the life cycle. It also incentivized engineers to focus on noise reduction, signal actionability, and other intra-stage improvements. This helps ensure not only that signals are detected, but also that the relevant engineers are able to quickly fix the issues before the code moves to the next stage of the development life cycle.</span></p>
|
||
<p><span style="font-weight: 400;">By evaluating progress against CPD, the team was able to take a quantitative approach to tracking projects within each stage to help reduce the engineering time spent fixing regressions.</span></p>
|
||
<figure id="attachment_17274" aria-describedby="caption-attachment-17274" style="width: 2000px" class="wp-caption alignnone"><img decoding="async" class="size-full wp-image-17274" src="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-02_FINAL.jpg" alt="Cost-per-developer (CPD) decreased throughout 2020 as Fix Fast-related improvements were implemented." width="2000" height="991" srcset="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-02_FINAL.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-02_FINAL.jpg?resize=916,454 916w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-02_FINAL.jpg?resize=768,381 768w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-02_FINAL.jpg?resize=1024,507 1024w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-02_FINAL.jpg?resize=1536,761 1536w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-02_FINAL.jpg?resize=96,48 96w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-02_FINAL.jpg?resize=192,95 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-17274" class="wp-caption-text">Cost-per-developer (CPD) decreased throughout 2020 as Fix Fast-related improvements were implemented.</figcaption></figure>
|
||
<h2><span style="font-weight: 400;">Project playbook</span></h2>
|
||
<p><span style="font-weight: 400;">Once CPD was in place, the Fix Fast team developed a playbook of approaches that had made proven improvements in shifting left, noise reduction, actionability, and more.</span></p>
|
||
<h3><span style="font-weight: 400;">Shift left</span></h3>
|
||
<p><span style="font-weight: 400;">Earlier detection is by far the number one way to reduce fix times. Automated tests are important to ensure that regressions are not promoted into production. However, running tests in continuous integration and production require a significant amount of data center capacity and time. Given the sheer number of diffs that need to be tested each day, and the number of tests to choose from, we’ve employed </span><a href="https://engineering.fb.com/2018/11/21/developer-tools/predictive-test-selection/"><span style="font-weight: 400;">complex heuristics and machine learning algorithms</span></a><span style="font-weight: 400;"> to help choose the best tests to run at each stage, taking into account the amount of time and capacity available. On a typical day, Facebook runs millions of tests on diffs alone.</span></p>
|
||
<p><span style="font-weight: 400;">We are also developing automated tests within the IDE to allow engineers to run a subset of tests while they are coding, before they even submit their code for review. IDE test results are returned in minutes. By taking this approach, engineers can now get test signals 90 percent sooner, and we have decreased the number of post-commit failures by more than 10 percent.</span></p>
|
||
<h3><span style="font-weight: 400;">Signal quality</span></h3>
|
||
<p><span style="font-weight: 400;">Reducing noisy, duplicated, or unactionable signals is important for ensuring that engineers know which signals are important and can quickly fix any underlying regressions.</span></p>
|
||
<p><span style="font-weight: 400;">To address this, we first had to understand which signals were resulting in useful outcomes. After manually analyzing hundreds of such signals, we developed a heuristic, called meaningful action, to represent tasks that were useful and resulted in some action by an engineer. The meaningful action heuristic considers human comments, diffs attached to a task, or other nontrivial actions. By systematically removing the unhealthy bots (those more likely to produce noisy or otherwise inactionable signals), we were able to dedupe signals and map them to the right regression metrics with a reasonable threshold for triggering. Accurate and actionable information was added to the tasks to give users clear direction on issue fixing.</span></p>
|
||
<p><span style="font-weight: 400;">One example of this is </span><a href="https://engineering.fb.com/2020/03/05/developer-tools/incident-tracker/"><span style="font-weight: 400;">Health Compass</span></a><span style="font-weight: 400;">, where we successfully improved the meaningful action ratio by 20 percent last year by reducing noise and tuning the regression detection logic.</span></p>
|
||
<figure id="attachment_17275" aria-describedby="caption-attachment-17275" style="width: 2000px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="size-full wp-image-17275" src="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-03_FINAL.jpg" alt="Improvements to Health Compass regression detection quality as showcased by a reduction in “noisy” tasks." width="2000" height="999" srcset="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-03_FINAL.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-03_FINAL.jpg?resize=916,458 916w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-03_FINAL.jpg?resize=768,384 768w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-03_FINAL.jpg?resize=1024,511 1024w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-03_FINAL.jpg?resize=1536,767 1536w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-03_FINAL.jpg?resize=96,48 96w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-03_FINAL.jpg?resize=192,96 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-17275" class="wp-caption-text">Improvements to Health Compass regression detection quality as showcased by a reduction in “noisy” tasks.</figcaption></figure>
|
||
<h3><span style="font-weight: 400;">Faster attribution</span></h3>
|
||
<p><span style="font-weight: 400;">Getting the signal in front of the right person the first time can shave days off of fix time. But this is often easier said than done. When a task is assigned to the wrong owner, it can ping-pong around the company before finally landing with someone who can fix it. In internal testing, we found that for tasks that had three or more owners before being resolved, close to 50 percent of the task’s cycle time was spent on ping-pong reassignments before reaching the correct owner.</span></p>
|
||
<p><span style="font-weight: 400;">One win was achieved by simply adding an “I’m not the right owner” button to the task. Previously, engineers had to track down the next best owner themselves, which could create a significant delay and prevent them from moving on to a new task. Now, when an engineer indicates that they are not the right person to fix an issue, the ownership selection algorithm engages the next best prediction on whom to assign the task. This data is also being used to help improve our ownership selection algorithm over time. </span></p>
|
||
<p><span style="font-weight: 400;">Better regression triage and root cause analysis are important to solving this. One of the biggest success stories in this area is our multisect service. Multisect analyzes a branch to trace the root cause back to the specific diff where a regression was introduced. This year, we worked on speeding up this process. As a result, we can now find the right diff three times faster than before.</span></p>
|
||
<h3><span style="font-weight: 400;">Get clean, stay clean</span></h3>
|
||
<p><span style="font-weight: 400;">Implement a land-blocking signal to help prevent regressions from being promoted. Null pointer exceptions (NPE) are a common crash type that affect hundreds of millions of Android users of Facebook’s family of apps. Making Java classes NullSafe is an important step toward reducing and eliminating these crashes. </span></p>
|
||
<p><span style="font-weight: 400;">Our data has shown that non-NullSafe Java files are three times more likely to result in NPE crashes than Java files that are NullSafe. The more NullSafe issues a file has, the more likely it is to be associated with NPE crashes. In 2020, we applied static analysis to help us detect code that’s not yet NullSafe and to provide engineers with tools to rapidly fix these issues. But perhaps more important, once a class has no more NullSafe issues, it is marked with an @NullSafe annotation. This blocks the code from being promoted again if any new NullSafe issues are introduced. </span></p>
|
||
<figure id="attachment_17276" aria-describedby="caption-attachment-17276" style="width: 2000px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="size-full wp-image-17276" src="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-04_FINAL.jpg" alt="The more NullSafe issues present in a given Java file, the higher the likelihood that file will contribute to an NPE crash." width="2000" height="1126" srcset="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-04_FINAL.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-04_FINAL.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-04_FINAL.jpg?resize=916,516 916w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-04_FINAL.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-04_FINAL.jpg?resize=1024,577 1024w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-04_FINAL.jpg?resize=1536,865 1536w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-04_FINAL.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_V3-04_FINAL.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-17276" class="wp-caption-text">The more NullSafe issues present in a given Java file, the higher the likelihood that file will contribute to an NPE crash.</figcaption></figure>
|
||
<h2>What’s next?</h2>
|
||
<p><span style="font-weight: 400;">By implementing all of these procedures, we are able to detect a greater percentage of regressions earlier in the engineering life cycle, thereby reducing the overall effort required to fix these regressions. Additional work in machine learning and other techniques is ongoing, to monitor and refine the best mixture of early detection techniques at scale.</span></p>
|
||
<p><span style="font-weight: 400;">We have also established a robust set of metrics, which help ensure that the quality of regression signals being delivered to engineers is sufficient for them to quickly triage, diagnose, and repair underlying regressions without overwhelming them with potentially noisy or duplicative signals. In the near future, we will be investing in systems that can automatically tune the sensitivity of regression detectors without requiring an engineer to proactively monitor these metrics. For example, a detector that is producing potentially noisy or inactionable signals can be temporarily disabled until an engineer can diagnose the problem, thus preventing unnecessary noise from being generated.</span></p>
|
||
<p><span style="font-weight: 400;">Ultimately, these techniques have proved beneficial not only for engineers, who reduce the amount of time they spend fixing regressions, but also for customers, who benefit from fewer regressions and faster fixes.</span></p>
|
||
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-facebook"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-facebook-17271"
|
||
class="share-facebook sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/?share=facebook"
|
||
target="_blank"
|
||
aria-labelledby="sharing-facebook-17271"
|
||
>
|
||
<span id="sharing-facebook-17271" hidden>Share on Facebook (Opens in new window)</span>
|
||
<span>Facebook</span>
|
||
</a></li><li class="share-threads"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-threads-17271"
|
||
class="share-threads sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/?share=threads"
|
||
target="_blank"
|
||
aria-labelledby="sharing-threads-17271"
|
||
>
|
||
<span id="sharing-threads-17271" hidden>Share on Threads (Opens in new window)</span>
|
||
<span>Threads</span>
|
||
</a></li><li class="share-jetpack-whatsapp"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-whatsapp-17271"
|
||
class="share-jetpack-whatsapp sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/?share=jetpack-whatsapp"
|
||
target="_blank"
|
||
aria-labelledby="sharing-whatsapp-17271"
|
||
>
|
||
<span id="sharing-whatsapp-17271" hidden>Share on WhatsApp (Opens in new window)</span>
|
||
<span>WhatsApp</span>
|
||
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-linkedin-17271"
|
||
class="share-linkedin sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/?share=linkedin"
|
||
target="_blank"
|
||
aria-labelledby="sharing-linkedin-17271"
|
||
>
|
||
<span id="sharing-linkedin-17271" hidden>Share on LinkedIn (Opens in new window)</span>
|
||
<span>LinkedIn</span>
|
||
</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-reddit-17271"
|
||
class="share-reddit sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/?share=reddit"
|
||
target="_blank"
|
||
aria-labelledby="sharing-reddit-17271"
|
||
>
|
||
<span id="sharing-reddit-17271" hidden>Share on Reddit (Opens in new window)</span>
|
||
<span>Reddit</span>
|
||
</a></li><li class="share-x"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-x-17271"
|
||
class="share-x sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/?share=x"
|
||
target="_blank"
|
||
aria-labelledby="sharing-x-17271"
|
||
>
|
||
<span id="sharing-x-17271" hidden>Share on X (Opens in new window)</span>
|
||
<span>X</span>
|
||
</a></li><li class="share-bluesky"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-bluesky-17271"
|
||
class="share-bluesky sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/?share=bluesky"
|
||
target="_blank"
|
||
aria-labelledby="sharing-bluesky-17271"
|
||
>
|
||
<span id="sharing-bluesky-17271" hidden>Share on Bluesky (Opens in new window)</span>
|
||
<span>Bluesky</span>
|
||
</a></li><li class="share-mastodon"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-mastodon-17271"
|
||
class="share-mastodon sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/?share=mastodon"
|
||
target="_blank"
|
||
aria-labelledby="sharing-mastodon-17271"
|
||
>
|
||
<span id="sharing-mastodon-17271" hidden>Share on Mastodon (Opens in new window)</span>
|
||
<span>Mastodon</span>
|
||
</a></li><li class="share-custom share-custom-hackernews"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-custom-17271"
|
||
class="share-custom share-custom-hackernews sd-button share-icon"
|
||
href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/?share=custom-1699562127"
|
||
target="_blank"
|
||
aria-labelledby="sharing-custom-17271"
|
||
>
|
||
<span id="sharing-custom-17271" hidden>Share on Hacker News (Opens in new window)</span>
|
||
<span style="background-image:url("https://news.ycombinator.com/y18.svg");">Hacker News</span>
|
||
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-email-17271"
|
||
class="share-email sd-button share-icon"
|
||
href="mailto:?subject=%5BShared%20Post%5D%20Faster%2C%20more%20efficient%20systems%20for%20finding%20and%20fixing%20regressions&body=https%3A%2F%2Fengineering.fb.com%2F2021%2F02%2F17%2Fdeveloper-tools%2Ffix-fast%2F&share=email"
|
||
target="_blank"
|
||
aria-labelledby="sharing-email-17271"
|
||
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you're having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="5b5fe86c30" data-email-share-track-url="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/?share=email">
|
||
<span id="sharing-email-17271" hidden>Email a link to a friend (Opens in new window)</span>
|
||
<span>Email</span>
|
||
</a></li><li class="share-end"></li></ul></div></div></div>
|
||
|
||
</div><!-- .entry-content -->
|
||
|
||
|
||
</article><!-- #post-## -->
|
||
|
||
<footer class="entry-footer">
|
||
|
||
|
||
</footer><!-- .entry-footer -->
|
||
|
||
<div class="fb-like"
|
||
data-href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/"
|
||
data-width=""
|
||
data-layout="standard"
|
||
data-action="like"
|
||
data-size="small"
|
||
data-share="true">
|
||
</div>
|
||
|
||
|
||
<div class="related-articles">
|
||
<div class="related-articles__header">
|
||
<h3 class="related-articles__header__headline">Read More in DevInfra</h3>
|
||
<a
|
||
aria-label="View all articles in this DevInfra"
|
||
class="related-articles__header__link"
|
||
href="https://engineering.fb.com/category/developer-tools/"
|
||
>
|
||
View All </a>
|
||
</div>
|
||
|
||
<div class="article-grids related-articles__grid">
|
||
<div class="row">
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17271 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-24394">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/Meta-MTIA-300-Hero-final.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Meta MTIA 300 - Hero - final" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-08-24T10:45:52-07:00">
|
||
AUG 24, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/mtia-300-meta-training-chip-built-in-nics/" rel="bookmark" aria-label="Read More about MTIA 300: Meta’s First Training Chip with Built-in NICs and Communication-Offloading Engines">
|
||
MTIA 300: Meta’s First Training Chip with Built-in NICs and Communication-Offloading Engines </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17271 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-24128">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/10-Years-Meta-Python-HERO-large.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-06-30T09:00:46-07:00">
|
||
JUN 30, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/06/30/open-source/10-years-of-metas-commitment-to-python/" rel="bookmark" aria-label="Read More about 10 Years of Meta’s Commitment to Python">
|
||
10 Years of Meta’s Commitment to Python </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17271 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-23898">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/capacity_efficiency_hero_white_option_5_1775676974.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="capacity_efficiency_hero_white_option_5_1775676974" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-04-16T09:00:34-07:00">
|
||
APR 16, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/04/16/developer-tools/capacity-efficiency-at-meta-how-unified-ai-agents-optimize-performance-at-hyperscale/" rel="bookmark" aria-label="Read More about Capacity Efficiency at Meta: How Unified AI Agents Optimize Performance at Hyperscale">
|
||
Capacity Efficiency at Meta: How Unified AI Agents Optimize Performance at Hyperscale </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17271 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-23856">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Escaping-the-Fork-Hero-2.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Escaping the Fork - Hero 2" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-04-09T09:00:34-07:00">
|
||
APR 9, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/04/09/developer-tools/escaping-the-fork-how-meta-modernized-webrtc-across-50-use-cases/" rel="bookmark" aria-label="Read More about Escaping the Fork: How Meta Modernized WebRTC Across 50+ Use Cases">
|
||
Escaping the Fork: How Meta Modernized WebRTC Across 50+ Use Cases </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17271 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-23871">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Meta-Tech-Podcast-episode-84-Configuration-Safety-at-Scale.webp?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-04-08T11:25:26-07:00">
|
||
APR 8, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/04/08/security/trust-but-canary-configuration-safety-at-scale-meta-tech-podcast/" rel="bookmark" aria-label="Read More about Trust But Canary: Configuration Safety at Scale">
|
||
Trust But Canary: Configuration Safety at Scale </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
<div class="article-grid col-12 col-md-4">
|
||
|
||
<article class="post post-17271 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-23840">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Compass-Not-Enclycopedia-Hero.png?w=580&h=326&crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
|
||
<header class="entry-header">
|
||
<div class="entry-header__top">
|
||
<span class="posted-on">
|
||
<time class="entry-date published" datetime="2026-04-06T09:00:50-07:00">
|
||
APR 6, 2026 </time>
|
||
</span>
|
||
</div>
|
||
|
||
<div class="entry-title">
|
||
<div>
|
||
<a href="https://engineering.fb.com/2026/04/06/developer-tools/how-meta-used-ai-to-map-tribal-knowledge-in-large-scale-data-pipelines/" rel="bookmark" aria-label="Read More about How Meta Used AI to Map Tribal Knowledge in Large-Scale Data Pipelines">
|
||
How Meta Used AI to Map Tribal Knowledge in Large-Scale Data Pipelines </a>
|
||
</div>
|
||
</div>
|
||
|
||
</header><!-- .entry-header -->
|
||
|
||
</article><!-- #post-## -->
|
||
</div><!-- /.article-grid -->
|
||
</div><!-- /.row -->
|
||
</div><!--/.article-grids -->
|
||
|
||
</div><!-- /.related-articles -->
|
||
|
||
|
||
<!-- include the carousel nav here. -->
|
||
|
||
|
||
|
||
|
||
</main><!-- #main -->
|
||
|
||
</div><!-- #primary -->
|
||
|
||
<!-- Do the right sidebar check. -->
|
||
|
||
|
||
|
||
|
||
<div class="col-md-4 widget-area" id="right-sidebar" role="complementary">
|
||
<aside id="fb_related_post_widget-4" class="widget widget_fb_related_post_widget"><h3 class="widget-title">Related Posts</h3><hr>
|
||
<div class="aside-related-projects"><div class="article-grid-redesign">
|
||
<article class="post-17180 post type-post status-publish has-post-thumbnail hentry" id="post-17180">
|
||
<a href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/" rel="bookmark" aria-label="Read More about Minesweeper automates root cause analysis as a first-line defense against bugs">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2021/02/Minesweeper-Hero.png?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2021/02/Minesweeper-Hero.png?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2021/02/Minesweeper-Hero.png?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Feb 09, 2021">Feb 09, 2021</time></span>
|
||
<h4 class="entry-title">Minesweeper automates root cause analysis as a first-line defense against bugs</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div><div class="article-grid-redesign">
|
||
<article class="post-16456 post type-post status-publish has-post-thumbnail hentry" id="post-16456">
|
||
<a href="https://engineering.fb.com/2020/08/07/security/pysa/" rel="bookmark" aria-label="Read More about Pysa: An open source static analysis tool to detect and prevent security issues in Python code">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2020/08/Zoncolon_heroBlueA-1.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2020/08/Zoncolon_heroBlueA-1.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2020/08/Zoncolon_heroBlueA-1.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Aug 07, 2020">Aug 07, 2020</time></span>
|
||
<h4 class="entry-title">Pysa: An open source static analysis tool to detect and prevent security issues in Python code</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div><div class="article-grid-redesign">
|
||
<article class="post-14109 post type-post status-publish has-post-thumbnail hentry" id="post-14109">
|
||
<a href="https://engineering.fb.com/2020/03/05/developer-tools/incident-tracker/" rel="bookmark" aria-label="Read More about Preventing performance regressions with Health Compass and Incident Tracker">
|
||
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2020/03/Health_Compass_01.jpg?w=580&h=326&crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Health_Compass_01.jpg?w=580&h=326&crop=1 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Health_Compass_01.jpg?w=580&h=326&crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
|
||
<header class="entry-header">
|
||
<span class="posted-on"><time class="entry-date published" datetime="Mar 05, 2020">Mar 05, 2020</time></span>
|
||
<h4 class="entry-title">Preventing performance regressions with Health Compass and Incident Tracker</h4>
|
||
</header><!-- .entry-header -->
|
||
</a>
|
||
</article><!-- #post-## -->
|
||
</div></div></aside><aside id="wpb_widget-3" class="widget widget_wpb_widget"><h3 class="widget-title">Related Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2316600655751692/>
|
||
<div class="job-description">
|
||
Machine Learning Engineer
|
||
</div>
|
||
<div class="job-location">
|
||
TEL AVIV, IL
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2887906508264541/>
|
||
<div class="job-description">
|
||
Software Engineer (Leadership) - Product
|
||
</div>
|
||
<div class="job-location">
|
||
LONDON, GB
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
SUNNYVALE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
BELLEVUE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
MENLO PARK, US
|
||
</div>
|
||
</a>
|
||
</li> </ul>
|
||
<a href="https://www.metacareers.com">
|
||
<div class="btn available-positions-btn">
|
||
See All Jobs </div>
|
||
</a>
|
||
</aside>
|
||
</div><!-- #secondary -->
|
||
|
||
|
||
|
||
<!-- <php get_template_part('content-single-side-bar'); ?> -->
|
||
|
||
<!-- Do the right sidebar check. -->
|
||
|
||
</div><!-- Container end -->
|
||
|
||
</div><!-- Wrapper end -->
|
||
|
||
</div>
|
||
|
||
<!-- ******************* The Footer Full-width Widget Area ******************* -->
|
||
|
||
<div class="wrapper wrapper-community-outer">
|
||
<div class="wrapper-community">
|
||
|
||
<div class="community" tabindex="-1">
|
||
|
||
<div class="row">
|
||
<!-- community section is generated through WordPress widget -->
|
||
<div id="wpb_widget-2" class="footer-widget widget_wpb_widget widget-count-3 col-md-4"><h3 class="widget-title">Available Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2316600655751692/>
|
||
<div class="job-description">
|
||
Machine Learning Engineer
|
||
</div>
|
||
<div class="job-location">
|
||
TEL AVIV, IL
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/2887906508264541/>
|
||
<div class="job-description">
|
||
Software Engineer (Leadership) - Product
|
||
</div>
|
||
<div class="job-location">
|
||
LONDON, GB
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
SUNNYVALE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
BELLEVUE, US
|
||
</div>
|
||
</a>
|
||
</li><li class="available-positions-item">
|
||
<a href=https://www.metacareers.com/jobs/1645503493182815/>
|
||
<div class="job-description">
|
||
Software Engineer, Machine Learning RecSys
|
||
</div>
|
||
<div class="job-location">
|
||
MENLO PARK, US
|
||
</div>
|
||
</a>
|
||
</li> </ul>
|
||
<a href="https://www.metacareers.com">
|
||
<div class="btn available-positions-btn">
|
||
See All Jobs </div>
|
||
</a>
|
||
</div><!-- .footer-widget --><div id="custom_html-2" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Technology at Meta</h3><div class="textwidget custom-html-widget"><ul class="stay-connected-lists">
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-fb-engineering" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Engineering at Meta - X
|
||
</div>
|
||
<a onclick='return twitterFollow("Meta_Engineers")'>
|
||
<button class="btn stay-connected-btn">
|
||
Follow
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-AI" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
AI at Meta
|
||
</div>
|
||
<a href="https://ai.meta.com/blog/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta Quest Blog
|
||
</div>
|
||
<a href="https://www.meta.com/blog/quest/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta for Developers
|
||
</div>
|
||
<a href="https://developers.facebook.com/">
|
||
<button class="btn stay-connected-btn">
|
||
Read
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
<li class="stay-connected-item">
|
||
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-bug-bounty" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
Meta Bug Bounty
|
||
</div>
|
||
<a href="https://bugbounty.meta.com/">
|
||
<button class="btn stay-connected-btn">
|
||
Learn more
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
|
||
<li class="stay-connected-item">
|
||
<img class="stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/rss.png" alt="footer-rss" loading="lazy" width="70" height="70">
|
||
<div class="stay-connected-item-right">
|
||
<div class="stay-connected-label">
|
||
RSS
|
||
</div>
|
||
<a href="https://code.facebook.com/posts/rss/">
|
||
<button class="btn stay-connected-btn">
|
||
Subscribe
|
||
</button>
|
||
</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div></div><!-- .footer-widget --><div id="custom_html-3" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Open Source</h3><div class="textwidget custom-html-widget"><p class="opensource-desc">
|
||
Meta believes in building community through open source technology. Explore our latest projects in Artificial Intelligence, Data Infrastructure, Development Tools, Front End, Languages, Platforms, Security, Virtual Reality, and more.
|
||
</p>
|
||
<ul class="opensource-cats">
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/android.png" alt="android" loading="lazy" width="50" height="59">
|
||
<div class="opensource-cap">
|
||
ANDROID
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/ios.png" alt="ios" loading="lazy" width="48" height="58">
|
||
<div class="opensource-cap">
|
||
iOS
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/web.png" alt="web" loading="lazy" width="55" height="49">
|
||
<div class="opensource-cap">
|
||
WEB
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/backend.png" alt="backend" loading="lazy" width="56" height="51">
|
||
<div class="opensource-cap">
|
||
BACKEND
|
||
</div>
|
||
</li>
|
||
<li class="opensource-cat">
|
||
<img src="/wp-content/themes/code-fb-com/img/hardware.png" alt="hardware" loading="lazy" width="55" height="44">
|
||
<div class="opensource-cap">
|
||
HARDWARE
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
|
||
<button class="btn opensource-btn" onclick="window.location.href='https://opensource.fb.com/'">
|
||
Learn More
|
||
</button>
|
||
|
||
</div></div><!-- .footer-widget -->
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</div><!-- #wrapper-footer-full -->
|
||
|
||
|
||
<div class="wrapper-footer" id="wrapper-footer">
|
||
<div class="inner-container">
|
||
<div class="footer">
|
||
<div class="row">
|
||
<div class="col-md-12">
|
||
<footer class="meta-footer" id="meta-footer">
|
||
<div class="site-info row justify-content-between">
|
||
<div class="col-md-5 meta-logo-container">
|
||
<a href="https://about.facebook.com/">
|
||
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/meta_logo_full.svg" width="89" height="18" alt="Meta" />
|
||
</a>
|
||
<div id="text-2" class="widget widget_text"> <div class="textwidget"><p>Engineering at Meta is a technical news resource for engineers interested in how we solve large-scale technical challenges at Meta.</p>
|
||
</div>
|
||
</div> </div>
|
||
<div class="col-md-6 footer-meta-links-container">
|
||
<ul class="meta-footer-links">
|
||
<li class="footer-link"><a href="https://engineering.fb.com">Home</a></li><li class="footer-link"><a href="https://about.meta.com/">Company Info</a></li><li class="footer-link"><a href="https://www.metacareers.com/?ref=engineering.fb.com">Careers</a></li> </ul>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="footer">
|
||
<div class="row">
|
||
|
||
<div class="col-md-12">
|
||
|
||
<footer class="site-footer" id="colophon">
|
||
<div class="inner-container">
|
||
<div class="site-info row justify-content-between">
|
||
|
||
<div class="col-md-4 copyright-container">
|
||
<div class="copyright"> © 2026 Meta</div>
|
||
</div>
|
||
|
||
<!-- static footer -->
|
||
<div class="col-md-7 footer-links-container">
|
||
<ul class="footer-links">
|
||
<li class="footer-link"><a href="https://www.facebook.com/policies">Terms</a></li><li class="footer-link"><a href="https://www.facebook.com/privacy/policy">Privacy</a></li><li class="footer-link"><a href="/privacy">Cookies</a></li><li class="footer-link"><a href="https://www.facebook.com/help">Help</a></li> </ul>
|
||
</div>
|
||
</div>
|
||
</div><!-- .site-info -->
|
||
|
||
</footer><!-- #colophon -->
|
||
|
||
</div><!--col end -->
|
||
|
||
</div><!-- row end -->
|
||
|
||
</div><!-- //.inner-container -->
|
||
</div><!-- wrapper end -->
|
||
|
||
</div><!-- #page we need this extra closing tag here -->
|
||
|
||
<script type="speculationrules">
|
||
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/code-fb-com/*","/wp-content/themes/understrap/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
|
||
</script>
|
||
<div class="gdprconsent-container" id="GDPRConsentBar">
|
||
<div class="gdprconsent-wrapper">
|
||
<div class="gdprconsent-content">
|
||
<p>To help personalize content, tailor and measure ads and provide a safer experience, we use cookies. By clicking or navigating the site, you agree to allow our collection of information on and off Facebook through cookies. Learn more, including about available controls: <a href="/privacy">Cookie Policy</a></p>
|
||
<div class="gdprconsent-button">
|
||
<button type="button" data-gdpr-accept>Accept</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Twitter universal website tag code -->
|
||
<script>
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
|
||
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
|
||
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
|
||
// Insert Twitter Pixel ID and Standard Event data below
|
||
twq('init','o31u0');
|
||
twq('track','PageView');
|
||
});
|
||
}
|
||
</script>
|
||
<!-- End Twitter universal website tag code -->
|
||
|
||
|
||
<script type="text/javascript">
|
||
window.WPCOM_sharing_counts = {"https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/":17271};
|
||
</script>
|
||
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KZRLGZ8" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
<noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=660742068067366&ev=PageView&noscript=1"/></noscript>
|
||
<script id="child-understrap-scripts-js-extra">
|
||
var loadmore_params = {"restfulURL":"https://engineering.fb.com/wp-json/fb/v1/loadmore","searchText":"","posts":"{\"page\":0,\"year\":2021,\"monthnum\":2,\"day\":17,\"name\":\"fix-fast\",\"category_name\":\"developer-tools\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","post_type":"post"};
|
||
var search_params = {"ajaxurl":"https://engineering.fb.com/wp-admin/admin-ajax.php","posts":"{\"page\":0,\"year\":2021,\"monthnum\":2,\"day\":17,\"name\":\"fix-fast\",\"category_name\":\"developer-tools\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","article_filter":"2f23ad5ce3"};
|
||
var more_articles = {"next_article":{"label":"Next","permalink":"https://engineering.fb.com/2021/02/22/production-engineering/foqs-scaling-a-distributed-priority-queue/","title":"FOQS: Scaling a distributed priority queue","thumbnail":["https://engineering.fb.com/wp-content/uploads/2021/02/CD21_047_ENGBlog_FOQS_HERO_FINAL.png?w=580&h=326&crop=1",580,326,true]},"prev_article":{"label":"Prev","permalink":"https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/","title":"Minesweeper automates root cause analysis as a first-line defense against bugs","thumbnail":["https://engineering.fb.com/wp-content/uploads/2021/02/Minesweeper-Hero.png?w=580&h=326&crop=1",580,326,true]}};
|
||
//# sourceURL=child-understrap-scripts-js-extra
|
||
</script>
|
||
<script type="text/javascript" src="https://engineering.fb.com/_static/??/wp-content/themes/code-fb-com/dist/js/child-theme.min.js,/wp-content/themes/code-fb-com/js/prism.js?m=1783017802j" ></script><script id="sharing-js-js-extra">
|
||
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":""};
|
||
//# sourceURL=sharing-js-js-extra
|
||
</script>
|
||
<script id="sharing-js-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/_inc/build/sharedaddy/sharing.min.js?ver=16.1.3"></script>
|
||
<script id="sharing-js-js-after">
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-facebook' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-threads' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-threads' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomthreads', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-linkedin' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-x' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-x' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomx', 'menubar=1,resizable=1,width=600,height=350' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-bluesky' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-bluesky' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcombluesky', 'menubar=1,resizable=1,width=600,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
var windowOpen;
|
||
( function () {
|
||
function matches( el, sel ) {
|
||
return !! (
|
||
el.matches && el.matches( sel ) ||
|
||
el.msMatchesSelector && el.msMatchesSelector( sel )
|
||
);
|
||
}
|
||
|
||
document.body.addEventListener( 'click', function ( event ) {
|
||
if ( ! event.target ) {
|
||
return;
|
||
}
|
||
|
||
var el;
|
||
if ( matches( event.target, 'a.share-mastodon' ) ) {
|
||
el = event.target;
|
||
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-mastodon' ) ) {
|
||
el = event.target.parentNode;
|
||
}
|
||
|
||
if ( el ) {
|
||
event.preventDefault();
|
||
|
||
// If there's another sharing window open, close it.
|
||
if ( typeof windowOpen !== 'undefined' ) {
|
||
windowOpen.close();
|
||
}
|
||
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcommastodon', 'menubar=1,resizable=1,width=460,height=400' );
|
||
return false;
|
||
}
|
||
} );
|
||
} )();
|
||
//# sourceURL=sharing-js-js-after
|
||
</script>
|
||
<script id="video-analytics-js-after">
|
||
if (window.gdprSafeTrack) {
|
||
window.gdprSafeTrack(function() {
|
||
let youtube = document.querySelectorAll('[src*="youtube-nocookie"]');
|
||
for (let i = 0; i < youtube.length; i++) {
|
||
youtube[i].src = youtube[i].src.replace('youtube-nocookie.com', 'youtube.com');
|
||
}
|
||
let vimeo = document.querySelectorAll('[src*="vimeo"]');
|
||
for (let i = 0; i < vimeo.length; i++) {
|
||
vimeo[i].src = vimeo[i].src.replace('&dnt=true', '');
|
||
}
|
||
});
|
||
}
|
||
//# sourceURL=video-analytics-js-after
|
||
</script>
|
||
<script id="wp-emoji-settings" type="application/json">
|
||
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://engineering.fb.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
|
||
</script>
|
||
<script type="module">
|
||
/*! This file is auto-generated */
|
||
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
|
||
//# sourceURL=https://engineering.fb.com/wp-includes/js/wp-emoji-loader.min.js
|
||
</script>
|
||
|
||
</body>
|
||
|
||
</html>
|