Files
nexus/sreweekly/articles/268/07-reverse-debugging-at-scale.html
2026-09-12 17:23:01 +08:00

1248 lines
107 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-title" content="Engineering at Meta - Engineering at Meta Blog">
<meta property="fb:app_id" content="1425766027653270" />
<link rel="profile" href="http://gmpg.org/xfn/11">
<link rel="pingback" href="https://engineering.fb.com/xmlrpc.php">
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
<script async src="https://engineering.fb.com/wp-content/themes/code-fb-com/dist/js/optimize.min.js"></script>
<!-- This site is optimized with the Yoast SEO Premium plugin v19.3 (Yoast SEO v27.3) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
<title>Reverse debugging at scale - Engineering at Meta</title>
<meta name="description" content="We developed a new reverse debugging technique allowing engineers to trace a run to find the root cause without rerunning the program." />
<link rel="canonical" href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/" />
<meta name="twitter:label1" content="Written by" />
<meta name="twitter:data1" content="Walter Erquinigo, David Carrillo-Cisneros, Alston Tang, changigi649" />
<meta name="twitter:label2" content="Est. reading time" />
<meta name="twitter:data2" content="8 minutes" />
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/#article","isPartOf":{"@id":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/"},"author":[{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/d08032cd73d66973e8ade7161de1d9a4"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/b8b8791aa5ffe2f783bd301a6281d1ab"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/bda5a29a17d0d2d0599ba3e151f55e8c"},{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/7167c692371a46d5bb5879bbd971c6c0"}],"headline":"Reverse debugging at scale","datePublished":"2021-04-27T15:00:32+00:00","mainEntityOfPage":{"@id":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/"},"wordCount":1456,"publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"image":{"@id":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/04\/CD21_175_ENGBlog_ReverseDebugging_HERO_FINAL.jpg","articleSection":["DevInfra"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/","url":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/","name":"Reverse debugging at scale - Engineering at Meta","isPartOf":{"@id":"https:\/\/engineering.fb.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/#primaryimage"},"image":{"@id":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/04\/CD21_175_ENGBlog_ReverseDebugging_HERO_FINAL.jpg","datePublished":"2021-04-27T15:00:32+00:00","description":"We developed a new reverse debugging technique allowing engineers to trace a run to find the root cause without rerunning the program.","breadcrumb":{"@id":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/#primaryimage","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/04\/CD21_175_ENGBlog_ReverseDebugging_HERO_FINAL.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2021\/04\/CD21_175_ENGBlog_ReverseDebugging_HERO_FINAL.jpg","width":1920,"height":1080,"caption":"Reverse debugging at scale"},{"@type":"BreadcrumbList","@id":"https:\/\/engineering.fb.com\/2021\/04\/27\/developer-tools\/reverse-debugging\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/engineering.fb.com\/"},{"@type":"ListItem","position":2,"name":"Reverse debugging at scale"}]},{"@type":"WebSite","@id":"https:\/\/engineering.fb.com\/#website","url":"https:\/\/engineering.fb.com\/","name":"Engineering at Meta","description":"Engineering at Meta Blog","publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/engineering.fb.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/engineering.fb.com\/#organization","name":"Meta","url":"https:\/\/engineering.fb.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","width":29011,"height":12501,"caption":"Meta"},"image":{"@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/Engineering\/","https:\/\/x.com\/fb_engineering"]},[],{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/d08032cd73d66973e8ade7161de1d9a4","name":"Walter Erquinigo","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Walter Erquinigo"},"url":"https:\/\/engineering.fb.com\/author\/walter-erquinigo\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/b8b8791aa5ffe2f783bd301a6281d1ab","name":"David Carrillo-Cisneros","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"David Carrillo-Cisneros"},"url":"https:\/\/engineering.fb.com\/author\/david-carrillo-cisneros\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/bda5a29a17d0d2d0599ba3e151f55e8c","name":"Alston Tang","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g23494c9101089ad44ae88ce9d2f56aac","url":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/?s=96&d=mm&r=g","caption":"Alston Tang"},"url":"https:\/\/engineering.fb.com\/author\/alston-tang\/"},{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/7167c692371a46d5bb5879bbd971c6c0","name":"changigi649","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/a090692c0af9b5ed6f8071e0a05e3147f0ba61b52a51c0671d073d9adb4e198e?s=96&d=mm&r=g8fd1e2b34e75c09c4aea578e73cbde79","url":"https:\/\/secure.gravatar.com\/avatar\/a090692c0af9b5ed6f8071e0a05e3147f0ba61b52a51c0671d073d9adb4e198e?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/a090692c0af9b5ed6f8071e0a05e3147f0ba61b52a51c0671d073d9adb4e198e?s=96&d=mm&r=g","caption":"changigi649"},"url":"https:\/\/engineering.fb.com\/author\/changigi649\/"}]}</script>
<!-- / Yoast SEO Premium plugin. -->
<link rel='dns-prefetch' href='//secure.gravatar.com' />
<link rel='dns-prefetch' href='//v0.wordpress.com' />
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta &raquo; Feed" href="https://engineering.fb.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta &raquo; Comments Feed" href="https://engineering.fb.com/comments/feed/" />
<script async src="https://www.googleoptimize.com/optimize.js?id=OPT-TPNDQC5" onerror="dataLayer.hide.end && dataLayer.hide.end()"></script>
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2021%2F04%2F27%2Fdeveloper-tools%2Freverse-debugging%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2021%2F04%2F27%2Fdeveloper-tools%2Freverse-debugging%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='all-css-12' href='https://engineering.fb.com/_static/??-eJyNjUsOwjAMRC9EaiqghQXiKFVqhzSQn+pEiNuTFJBAbNjZTzNv4BYFBp+UTxBt1sYzFKQpzpVz5WQ4QSUv0CDzCj6KaVJOMWAgJc5jwe7ZKTnAyVgSS6Jxxv90XRbv3YtKUeJVtF3TgguUbZHyJGdFkui+nMbrvxWD8QhjLvvAAY20wgYd+OupspM7tv3+sO533Wb7AImQY4U=' type='text/css' media='all' />
<script id="gdprconsent-js" src="https://engineering.fb.com/wp-content/plugins/wp-gdpr-consent/dist/gdprconsent.js?ver=2.0.0"></script>
<script id="gdprconsent-js-after">
if (window.gdprSafeTrack) {
window.gdprSafeTrack('https://www.googletagmanager.com/gtag/js?id=391063774');
window.gdprSafeTrack(function() {
var google_analytics_g4 = decodeURIComponent( '391063774' );
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', google_analytics_g4);
});
}
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
var google_tag_manager = decodeURIComponent( 'GTM-KZRLGZ8' );
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer',google_tag_manager);
});
}
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
var facebook_pixel = decodeURIComponent( '660742068067366' );
!function(f,b,e,v,n,t,s)
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
n.queue=[];t=b.createElement(e);t.async=!0;
t.src=v;s=b.getElementsByTagName(e)[0];
s.parentNode.insertBefore(t,s)}(window, document,'script',
'https://connect.facebook.net/en_US/fbevents.js');
fbq('init', facebook_pixel);
fbq('track', 'PageView');
});
}
//# sourceURL=gdprconsent-js-after
</script>
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJzTLy/QzcxLzilNSS3WzwKiwtLUokoopZebmaeXVayjj0+Rbm5melFiSSpUsX2uraG5hbmZgYmloUkWAK+3Iic=" ></script><script type="text/javascript" src="https://engineering.fb.com/wp-content/themes/understrap/js/popper.min.js?m=1773259670g" ></script><link rel="https://api.w.org/" href="https://engineering.fb.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.fb.com/wp-json/wp/v2/posts/17484" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.fb.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress 7.1" />
<link rel='shortlink' href='https://wp.me/pa0Lhq-4y0' />
<link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/Optimistic_Display_W_Md.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="Reverse debugging at scale" />
<meta property="og:url" content="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/" />
<meta property="og:description" content="Say you receive an email notification that a service is crashing just after your last code change deploys. The crash happens in only 0.1 percent of the servers where it runs. But you’re at a large-…" />
<meta property="article:published_time" content="2021-04-27T15:00:32+00:00" />
<meta property="article:modified_time" content="2021-04-26T18:24:38+00:00" />
<meta property="og:site_name" content="Engineering at Meta" />
<meta property="og:image" content="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_HERO_FINAL.jpg" />
<meta property="og:image:width" content="1920" />
<meta property="og:image:height" content="1080" />
<meta property="og:image:alt" content="Reverse debugging at scale" />
<meta property="og:locale" content="en_US" />
<meta name="twitter:text:title" content="Reverse debugging at scale" />
<meta name="twitter:image" content="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_HERO_FINAL.jpg?w=640" />
<meta name="twitter:image:alt" content="Reverse debugging at scale" />
<meta name="twitter:card" content="summary_large_image" />
<!-- End Jetpack Open Graph Tags -->
<style id="wp-custom-css">
.table-of-contents {
position: sticky;
top: 20px;
background: #fff;
padding: 1em;
border: 1px solid #ddd;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
max-width: 250px;
}
</style>
<link rel="shortcut icon" href="https://engineering.fb.com/wp-content/themes/code-fb-com/favicon.ico" />
</head>
<body class="wp-singular post-template-default single single-post postid-17484 single-format-standard wp-theme-understrap wp-child-theme-code-fb-com group-blog">
<div class="hfeed site" id="page">
<!-- ******************* The Navbar Area ******************* -->
<div class="wrapper-fluid wrapper-navbar" id="wrapper-navbar">
<a class="skip-link screen-reader-text sr-only" href="#content">
Skip to content </a>
<nav class="navbar navbar-expand-std">
<div class="nav-container">
<!-- Your site title as branding in the menu -->
<div class="nav-top-row">
<h1 class="navbar-brand mb-0">
<a rel="home" href="https://engineering.fb.com/" title="Engineering at Meta">
<img class="navbar-brand-logo" src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/logo-meta.svg" class="navbar-brand-logo" alt="Engineering at Meta" width="215" height="40" loading="lazy">
</a>
</h1>
<!-- end custom logo -->
<label for="search" class="screen-reader-text">Search this site</label>
<input type="text" id="search" name="search" class="form-control-lg navbar-search" autocomplete="off">
<button class="navbar-toggler navbar-searchbar-toggler"
type="button"
data-toggle="collapse"
data-target="#searchbarDropdown"
aria-controls="searchbarDropdown"
aria-expanded="false"
aria-label="Toggle navigation">
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/icon-search.svg" class="navbar-mobile-search-icon" loading="lazy" width="24" height="25" />
</button>
<!-- desktop language menu (mobile language menu is implemented via jQuery)-->
<div class="notranslate nice-select navbar-language form-control">
</div>
<button class="navbar-toggler collapsed navbar-menu-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<div class="navbar-border"></div>
<!-- The WordPress Menu goes here -->
<div id="navbarNavDropdown" class="collapse navbar-collapse"><ul id="main-menu" class="navbar-nav"><li id="menu-item-60" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-60 dropdown"><a title="Open Source" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Open Source <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-12330" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-12330"><a title="Open Source" href="https://engineering.fb.com/category/open-source/" class="nav-link">Open Source</a></li>
<li id="menu-item-12331" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-12331"><a title="Meta Open Source" href="https://opensource.fb.com" class="nav-link">Meta Open Source</a></li>
</ul>
</li>
<li id="menu-item-2686" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2686 dropdown"><a title="Platforms" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Platforms <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-65" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-65"><a title="Android" href="https://engineering.fb.com/category/android/" class="nav-link">Android</a></li>
<li id="menu-item-66" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-66"><a title="iOS" href="https://engineering.fb.com/category/ios/" class="nav-link">iOS</a></li>
<li id="menu-item-67" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-67"><a title="Web" href="https://engineering.fb.com/category/web/" class="nav-link">Web</a></li>
</ul>
</li>
<li id="menu-item-2682" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2682 dropdown"><a title="Infrastructure Systems" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Infrastructure Systems <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-218" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-218"><a title="Core Infra" href="https://engineering.fb.com/category/core-infra/" class="nav-link">Core Infra</a></li>
<li id="menu-item-219" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-219"><a title="Data Infrastructure" href="https://engineering.fb.com/category/data-infrastructure/" class="nav-link">Data Infrastructure</a></li>
<li id="menu-item-220" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-220"><a title="DevInfra" href="https://engineering.fb.com/category/developer-tools/" class="nav-link">DevInfra</a></li>
<li id="menu-item-226" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-226"><a title="Production Engineering" href="https://engineering.fb.com/category/production-engineering/" class="nav-link">Production Engineering</a></li>
<li id="menu-item-227" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-227"><a title="Security &amp; Privacy" href="https://engineering.fb.com/category/security/" class="nav-link">Security &amp; Privacy</a></li>
<li id="menu-item-18607" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18607"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/systems-infrastructure/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2683" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2683 dropdown"><a title="Physical Infrastructure" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Physical Infrastructure <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-215" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-215"><a title="Connectivity" href="https://engineering.fb.com/category/connectivity/" class="nav-link">Connectivity</a></li>
<li id="menu-item-216" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-216"><a title="Data Center Engineering" href="https://engineering.fb.com/category/data-center-engineering/" class="nav-link">Data Center Engineering</a></li>
<li id="menu-item-217" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-217"><a title="Networking &amp; Traffic" href="https://engineering.fb.com/category/networking-traffic/" class="nav-link">Networking &amp; Traffic</a></li>
<li id="menu-item-18608" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18608"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/networking-connectivity/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2684" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2684 dropdown"><a title="Video Engineering &amp; AR/VR" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Video Engineering &amp; AR/VR <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-225" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-225"><a title="Video Engineering" href="https://engineering.fb.com/category/video-engineering/" class="nav-link">Video Engineering</a></li>
<li id="menu-item-223" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-223"><a title="Virtual Reality" href="https://engineering.fb.com/category/virtual-reality/" class="nav-link">Virtual Reality</a></li>
<li id="menu-item-13388" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13388"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/augmented-reality-virtual-reality/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2685" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2685 dropdown"><a title="Artificial Intelligence" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Artificial Intelligence <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-222" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-222"><a title="ML Applications" href="https://engineering.fb.com/category/ml-applications/" class="nav-link">ML Applications</a></li>
<li id="menu-item-221" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-221"><a title="AI Research" href="https://engineering.fb.com/category/ai-research/" class="nav-link">AI Research</a></li>
<li id="menu-item-13387" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13387"><a title="Research Publications" href="https://ai.facebook.com/results/?content_types%5B0%5D=publication" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-228" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-228"><a title="Watch Videos" href="/videos" class="nav-link">Watch Videos</a></li>
</ul></div> </div><!-- .container -->
<div class="search-container collapse" id="searchbarDropdown">
<div class="searchbar-search">
<div class="searchbar-container">
<div class="navbar-mobile-search-margin"></div>
<input type="text" name="search" placeholder="Search" class="form-control-lg navbar-mobile-search" autofocus autocomplete="off">
<button class="searchbar-clear">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
</div>
</div>
</nav><!-- .site-navigation -->
</div><!-- .wrapper-navbar end -->
<div class="wrapper" id="single-wrapper">
<div class="single-container row" id="content" tabindex="-1">
<!-- Do the left sidebar check. -->
<div class="content-area" id="primary">
<main class="site-main" id="main">
<article class="post-17484 post type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-17484">
<header class="entry-header">
<div class="entry-meta">
POSTED ON <span class="posted-on entry-date"><time class="published updated" datetime="2021-04-27">APRIL 27, 2021</time></span> TO <span class="cat-links"><a href="https://engineering.fb.com/category/developer-tools/" rel="category tag">DevInfra</a></span>
<h1 class="entry-title"><div>Reverse debugging at scale</div></h1> </div><!-- .entry-meta -->
</header><!-- .entry-header -->
<figure id="post-feat-image-container">
<img width="1920" height="1080" src="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_HERO_FINAL.jpg" class="wp-post-image" alt="Reverse debugging at scale" title="CD21_175_ENGBlog_ReverseDebugging_HERO_v1" /> </figure>
<div class="entry-authors-social">
<div class="entry-authors">
By <a href="https://engineering.fb.com/author/walter-erquinigo/" title="Posts by Walter Erquinigo" class="author url fn" rel="author">Walter Erquinigo</a>, <a href="https://engineering.fb.com/author/david-carrillo-cisneros/" title="Posts by David Carrillo-Cisneros" class="author url fn" rel="author">David Carrillo-Cisneros</a>, <a href="https://engineering.fb.com/author/alston-tang/" title="Posts by Alston Tang" class="author url fn" rel="author">Alston Tang</a> </div>
</div>
<div class="entry-content">
<p><span style="font-weight: 400;">Say you receive an email notification that a service is crashing just after your last code change deploys. The crash happens in only 0.1 percent of the servers where it runs. But you’re at a large-scale company, so 0.1 percent equals thousands of servers — and this issue is going to be hard to reproduce. Several hours later, you still can’t reproduce it, and you have spent a full day chasing this issue.</span></p>
<p><span style="font-weight: 400;">This is where reverse debugging kicks in. Existing methods allow engineers to record a paused (or crashed) program, then rewind and replay to locate the root cause. However, for large-scale companies like Facebook, these solutions impose too much overhead to be practical in production. To address this, we developed a new technique that allows engineers to trace a failing run and inspect its history to find the root cause without the hassle of rerunning the program, which saves an enormous amount of time. We do this by efficiently tracing the CPU activity on our servers and, upon crashes, saving the process history, which is later displayed in a human-readable format with the help of the </span><a href="https://lldb.llvm.org/"><span style="font-weight: 400;">LLDB</span></a><span style="font-weight: 400;"> debugger, offering everything from instruction history views to reverse debugging.</span></p>
<h2><span style="font-weight: 400;">Sounds great, but how does it work?</span></h2>
<p><span style="font-weight: 400;">To help develop this, we leveraged </span><a href="https://software.intel.com/content/www/us/en/develop/blogs/processor-tracing.html"><span style="font-weight: 400;">Intel Processor Trace</span></a><span style="font-weight: 400;"> (Intel PT), which uses specialized hardware to accelerate the collection of the steps in a program, allowing us to use it in production and at scale. However, tracing a program efficiently is just the first step.</span></p>
<h3><span style="font-weight: 400;">Continuous collection and quick storage in production</span></h3>
<p><span style="font-weight: 400;">As we don’t know when a crash will happen nor which process will crash, we need to continuously collect an Intel PT trace of all running processes. In order to bound the memory required to operate, we store the trace in a circular buffer where new data overwrites old data.</span></p>
<figure id="attachment_17488" aria-describedby="caption-attachment-17488" style="width: 1920px" class="wp-caption alignnone"><img decoding="async" class="size-full wp-image-17488" src="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart1_v1.jpg" alt="When multiple processes (A and B) are running concurrently, the trace data is stored in the buffer. At t+8, Process B’s data begins to overwrite the oldest data (Process A’s) in the buffer." width="1920" height="1080" srcset="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart1_v1.jpg 1920w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart1_v1.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart1_v1.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart1_v1.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart1_v1.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart1_v1.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart1_v1.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart1_v1.jpg?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-17488" class="wp-caption-text">When multiple processes (A and B) are running concurrently, the trace data is stored in the buffer. At t+8, Process B’s data begins to overwrite the oldest data (Process A’s) in the buffer.</figcaption></figure>
<p><span style="font-weight: 400;">In large servers, like</span><a href="https://engineering.fb.com/2019/03/14/data-center-engineering/accelerating-infrastructure/"> <span style="font-weight: 400;">the ones used to train AI models</span></a><span style="font-weight: 400;">, Intel PT can generate hundreds of megabytes of data per second even in its compressed format. When a process crashes, the collector must pause the collection and copy the content of the buffer for the crashed process. To do this, the operating system needs to notify our collector that a process has crashed, which takes time. The larger the delay between the crash and the copy of the buffer content, the more data that will be overwritten by new data from other processes.</span></p>
<p><span style="font-weight: 400;">To address this, we designed an </span><a href="https://ebpf.io/"><span style="font-weight: 400;">eBPF</span></a><span style="font-weight: 400;"> kernel probe, which is a program that is triggered upon certain kernel events, to notify our collector within a few milliseconds that a crash happened, thus maximizing the amount of information in the trace relevant to the crash. We tried several approaches, but none was as fast as this one. An additional design consideration is that crashes often leave the computer in a bad state, which prevents us from doing any analysis on the collected trace on the fly. That’s why we decided to store the traces and the corresponding binaries of the crashed processes on our data centers, so that they can be analyzed later on a different machine.</span></p>
<h3><span style="font-weight: 400;">Decoding and symbolication</span></h3>
<p><span style="font-weight: 400;">How do we transform this data into instructions that can be analyzed? Raw traces are undecipherable to engineers. Humans need context such as source line information and function names. To add context, we built a component in LLDB that is able to receive a trace and reconstruct the instructions, along with their symbols. We recently started open-sourcing this work. You can find it </span><a href="https://github.com/llvm/llvm-project/tree/main/lldb/source/Plugins/Trace/intel-pt"><span style="font-weight: 400;">here</span></a><span style="font-weight: 400;">.</span></p>
<figure id="attachment_17487" aria-describedby="caption-attachment-17487" style="width: 1920px" class="wp-caption alignnone"><img decoding="async" class="size-full wp-image-17487" src="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart2_v1.jpg" alt="A trace primarily contains the information about which branches were taken and which weren’t. We compare that information with all the instructions from the original binary and reconstruct the instructions that were executed by the program. Later, with the help of LLDB’s symbolication stack, we can obtain the corresponding source code information and show it to the engineer in a readable fashion." width="1920" height="1080" srcset="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart2_v1.jpg 1920w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart2_v1.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart2_v1.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart2_v1.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart2_v1.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart2_v1.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart2_v1.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart2_v1.jpg?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-17487" class="wp-caption-text">A trace primarily contains the information about which branches were taken and which weren’t. We compare that information with all the instructions from the original binary and reconstruct the instructions that were executed by the program. Later, with the help of LLDB’s symbolication stack, we can obtain the corresponding source code information and show it to the engineer in a readable fashion.</figcaption></figure>
<p><span style="font-weight: 400;">With the flow mentioned in the picture above, we are able to convert raw trace file into the following:</span></p>
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-17489" src="https://engineering.fb.com/wp-content/uploads/2021/04/codeblock.jpg" alt="" width="1920" height="977" srcset="https://engineering.fb.com/wp-content/uploads/2021/04/codeblock.jpg 1920w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock.jpg?resize=916,466 916w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock.jpg?resize=768,391 768w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock.jpg?resize=1024,521 1024w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock.jpg?resize=1536,782 1536w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock.jpg?resize=96,49 96w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock.jpg?resize=192,98 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">Symbolicating the instructions is only the first part of LLDB’s work. The next part is reconstructing the function call history. For this, we built an algorithm that is able to walk through the instructions and construct a tree of function calls, i.e., which function invoked which other function and when. The most interesting part of it is that traces can start at an arbitrary moment in the program execution, and not necessarily at the beginning.</span></p>
<figure id="attachment_17490" aria-describedby="caption-attachment-17490" style="width: 1920px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="size-full wp-image-17490" src="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart5_v1.jpg" alt="Sample function call tree, where each vertical segment contains instructions and call sites are displayed with arrows." width="1920" height="1080" srcset="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart5_v1.jpg 1920w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart5_v1.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart5_v1.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart5_v1.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart5_v1.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart5_v1.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart5_v1.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart5_v1.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><figcaption id="caption-attachment-17490" class="wp-caption-text">Sample function call tree, where each vertical segment contains instructions and call sites are displayed with arrows.</figcaption></figure>
<p><span style="font-weight: 400;">This tree can help us quickly answer questions like what the stack trace is at a given point in history, which is solved simply by walking up the tree. On the other hand, a more complicated question is figuring out where a reverse-step-over stops.</span></p>
<p><span style="font-weight: 400;">Imagine that you are in line 16 of this little piece of code. If you reverse-step-over, you should stop at either line 13 or 15, depending on whether the </span><span style="font-weight: 400;">if </span><span style="font-weight: 400;">was taken or not. A naive approach would be to inspect the instructions in the history one by one until one of these lines is found, but this can be very inefficient, as the function </span><span style="font-weight: 400;">foo</span><span style="font-weight: 400;"> can include millions of instructions. Instead, we can use the aforementioned tree to move across lines without jumping into calls. Unlike the previous approach, traversing the tree is almost trivial.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-17492 " src="https://engineering.fb.com/wp-content/uploads/2021/04/Screen-Shot-2021-04-21-at-11.14.05-AM.png?w=580&amp;h=202&amp;crop=1" alt="" width="390" height="136" /></p>
<p><span style="font-weight: 400;">Besides, breakpoint support has also been added. Suppose you are in the middle of a debug session and you want to go back in time to the most recent call to </span><span style="font-weight: 400;">function_a</span><span style="font-weight: 400;">. Then you could do:</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-17511 size-full" src="https://engineering.fb.com/wp-content/uploads/2021/04/codeblock4.jpg" alt="" width="1920" height="841" srcset="https://engineering.fb.com/wp-content/uploads/2021/04/codeblock4.jpg 1920w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock4.jpg?resize=916,401 916w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock4.jpg?resize=768,336 768w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock4.jpg?resize=1024,449 1024w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock4.jpg?resize=1536,673 1536w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock4.jpg?resize=96,42 96w, https://engineering.fb.com/wp-content/uploads/2021/04/codeblock4.jpg?resize=192,84 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">Finally, we are planning to integrate this flow with VSCode at some point, for a rich visual reverse debugging experience.</span></p>
<h3><span style="font-weight: 400;">Latency analysis and profiling</span></h3>
<p><span style="font-weight: 400;">An execution trace contains more information than other representations of control flow, like call-stacks. Furthermore, as part of our traces we are collecting timing information with great accuracy, allowing engineers to know not only the sequence of operations (or functions) executed but also their timestamps, enabling a new use case: latency analysis.</span></p>
<p><span style="font-weight: 400;">Imagine a service that handles different requests that fetch data leveraging an internal cache. Under certain conditions, the cache must be flushed, and the next fetch call will take much longer.</span></p>
<p><span style="font-weight: 400;">The server receives several requests from engineers, and you want to understand the long tail of the latency in your service (e.g., P99). You get your profiler and collect some sampled call stacks for the main two types of requests. They look like the graph below, which does not tell you how they are different because call stack sampling shows only aggregated numbers. We need to convert traces into execution path info and symbolicate the instructions along the path.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-17494" src="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart3_FINAL.jpg" alt="Sampled call stacks for the main two types of requests, which do not tell you how they are different because call stack sampling shows only aggregated numbers. " width="1920" height="1080" srcset="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart3_FINAL.jpg 1920w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart3_FINAL.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart3_FINAL.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart3_FINAL.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart3_FINAL.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart3_FINAL.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart3_FINAL.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart3_FINAL.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">The image below shows a trace, making it easy to understand what’s happening: Request B flushes data before getting it.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-17495" src="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart4_FINAL.jpg" alt="Image showing a trace, making it easy to understand what’s happening: Request B flushes data before getting it." width="1920" height="1080" srcset="https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart4_FINAL.jpg 1920w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart4_FINAL.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart4_FINAL.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart4_FINAL.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart4_FINAL.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart4_FINAL.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart4_FINAL.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2021/04/CD21_175_ENGBlog_ReverseDebugging_Flowchart4_FINAL.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">While a debugger can help you move within the trace step-wise, a visualization can help you see the big picture easily. That’s why we are also working to add graphs like the “Call stack trace” above in our performance analysis tool,</span><a href="https://www.facebook.com/atscaleevents/videos/996197807391867/"> <span style="font-weight: 400;">Tracery</span></a><span style="font-weight: 400;">. We are building a tool that can produce this tracing and timing data, and combine it with the symbolication produced by LLDB. Our goal is to offer developers a way to see all this information at a glance, and then zoom in and out and get the data they need at the level they need.</span></p>
<p><span style="font-weight: 400;">Remember the undesirable scenario at the beginning of this post? Now imagine that you receive that email notification telling you that your service is crashing in 0.1 percent of machines, but this time it comes with a “Reverse debug on VSCode” button. You click it and then move around the history of the program until you find the function call that shouldn’t have happened or an if-clause that shouldn’t have executed, and get it fixed within a few minutes.</span></p>
<p><span style="font-weight: 400;">This kind of technology enables us to solve problems at a level we didn’t consider possible before. Most of our historic work in this space resulted in considerable overhead, which limited our ability to scale. We are now building debugging tools that interactive and visually can help developers solve production and development issues in a much more efficient way. This is an exciting endeavor that will help us scale even more.</span></p>
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-facebook"><a rel="nofollow noopener noreferrer"
data-shared="sharing-facebook-17484"
class="share-facebook sd-button share-icon"
href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/?share=facebook"
target="_blank"
aria-labelledby="sharing-facebook-17484"
>
<span id="sharing-facebook-17484" hidden>Share on Facebook (Opens in new window)</span>
<span>Facebook</span>
</a></li><li class="share-threads"><a rel="nofollow noopener noreferrer"
data-shared="sharing-threads-17484"
class="share-threads sd-button share-icon"
href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/?share=threads"
target="_blank"
aria-labelledby="sharing-threads-17484"
>
<span id="sharing-threads-17484" hidden>Share on Threads (Opens in new window)</span>
<span>Threads</span>
</a></li><li class="share-jetpack-whatsapp"><a rel="nofollow noopener noreferrer"
data-shared="sharing-whatsapp-17484"
class="share-jetpack-whatsapp sd-button share-icon"
href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/?share=jetpack-whatsapp"
target="_blank"
aria-labelledby="sharing-whatsapp-17484"
>
<span id="sharing-whatsapp-17484" hidden>Share on WhatsApp (Opens in new window)</span>
<span>WhatsApp</span>
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
data-shared="sharing-linkedin-17484"
class="share-linkedin sd-button share-icon"
href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/?share=linkedin"
target="_blank"
aria-labelledby="sharing-linkedin-17484"
>
<span id="sharing-linkedin-17484" hidden>Share on LinkedIn (Opens in new window)</span>
<span>LinkedIn</span>
</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
data-shared="sharing-reddit-17484"
class="share-reddit sd-button share-icon"
href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/?share=reddit"
target="_blank"
aria-labelledby="sharing-reddit-17484"
>
<span id="sharing-reddit-17484" hidden>Share on Reddit (Opens in new window)</span>
<span>Reddit</span>
</a></li><li class="share-x"><a rel="nofollow noopener noreferrer"
data-shared="sharing-x-17484"
class="share-x sd-button share-icon"
href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/?share=x"
target="_blank"
aria-labelledby="sharing-x-17484"
>
<span id="sharing-x-17484" hidden>Share on X (Opens in new window)</span>
<span>X</span>
</a></li><li class="share-bluesky"><a rel="nofollow noopener noreferrer"
data-shared="sharing-bluesky-17484"
class="share-bluesky sd-button share-icon"
href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/?share=bluesky"
target="_blank"
aria-labelledby="sharing-bluesky-17484"
>
<span id="sharing-bluesky-17484" hidden>Share on Bluesky (Opens in new window)</span>
<span>Bluesky</span>
</a></li><li class="share-mastodon"><a rel="nofollow noopener noreferrer"
data-shared="sharing-mastodon-17484"
class="share-mastodon sd-button share-icon"
href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/?share=mastodon"
target="_blank"
aria-labelledby="sharing-mastodon-17484"
>
<span id="sharing-mastodon-17484" hidden>Share on Mastodon (Opens in new window)</span>
<span>Mastodon</span>
</a></li><li class="share-custom share-custom-hackernews"><a rel="nofollow noopener noreferrer"
data-shared="sharing-custom-17484"
class="share-custom share-custom-hackernews sd-button share-icon"
href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/?share=custom-1699562127"
target="_blank"
aria-labelledby="sharing-custom-17484"
>
<span id="sharing-custom-17484" hidden>Share on Hacker News (Opens in new window)</span>
<span style="background-image:url(&quot;https://news.ycombinator.com/y18.svg&quot;);">Hacker News</span>
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
data-shared="sharing-email-17484"
class="share-email sd-button share-icon"
href="mailto:?subject=%5BShared%20Post%5D%20Reverse%20debugging%20at%20scale&#038;body=https%3A%2F%2Fengineering.fb.com%2F2021%2F04%2F27%2Fdeveloper-tools%2Freverse-debugging%2F&#038;share=email"
target="_blank"
aria-labelledby="sharing-email-17484"
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you&#039;re having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="6daefcbacc" data-email-share-track-url="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/?share=email">
<span id="sharing-email-17484" hidden>Email a link to a friend (Opens in new window)</span>
<span>Email</span>
</a></li><li class="share-end"></li></ul></div></div></div>
</div><!-- .entry-content -->
</article><!-- #post-## -->
<footer class="entry-footer">
</footer><!-- .entry-footer -->
<div class="fb-like"
data-href="https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/"
data-width=""
data-layout="standard"
data-action="like"
data-size="small"
data-share="true">
</div>
<div class="related-articles">
<div class="related-articles__header">
<h3 class="related-articles__header__headline">Read More in DevInfra</h3>
<a
aria-label="View all articles in this DevInfra"
class="related-articles__header__link"
href="https://engineering.fb.com/category/developer-tools/"
>
View All </a>
</div>
<div class="article-grids related-articles__grid">
<div class="row">
<div class="article-grid col-12 col-md-4">
<article class="post post-17484 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-24394">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/Meta-MTIA-300-Hero-final.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Meta MTIA 300 - Hero - final" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-08-24T10:45:52-07:00">
AUG 24, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/mtia-300-meta-training-chip-built-in-nics/" rel="bookmark" aria-label="Read More about MTIA 300: Meta&#8217;s First Training Chip with Built-in NICs and Communication-Offloading Engines">
MTIA 300: Meta&#8217;s First Training Chip with Built-in NICs and Communication-Offloading Engines </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-17484 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-24128">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/10-Years-Meta-Python-HERO-large.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-30T09:00:46-07:00">
JUN 30, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/30/open-source/10-years-of-metas-commitment-to-python/" rel="bookmark" aria-label="Read More about 10 Years of Meta’s Commitment to Python">
10 Years of Meta’s Commitment to Python </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-17484 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-23898">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/capacity_efficiency_hero_white_option_5_1775676974.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="capacity_efficiency_hero_white_option_5_1775676974" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-04-16T09:00:34-07:00">
APR 16, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/04/16/developer-tools/capacity-efficiency-at-meta-how-unified-ai-agents-optimize-performance-at-hyperscale/" rel="bookmark" aria-label="Read More about Capacity Efficiency at Meta: How Unified AI Agents Optimize Performance at Hyperscale">
Capacity Efficiency at Meta: How Unified AI Agents Optimize Performance at Hyperscale </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-17484 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-23856">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Escaping-the-Fork-Hero-2.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Escaping the Fork - Hero 2" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-04-09T09:00:34-07:00">
APR 9, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/04/09/developer-tools/escaping-the-fork-how-meta-modernized-webrtc-across-50-use-cases/" rel="bookmark" aria-label="Read More about Escaping the Fork: How Meta Modernized WebRTC Across 50+ Use Cases">
Escaping the Fork: How Meta Modernized WebRTC Across 50+ Use Cases </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-17484 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-23871">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Meta-Tech-Podcast-episode-84-Configuration-Safety-at-Scale.webp?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-04-08T11:25:26-07:00">
APR 8, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/04/08/security/trust-but-canary-configuration-safety-at-scale-meta-tech-podcast/" rel="bookmark" aria-label="Read More about Trust But Canary: Configuration Safety at Scale">
Trust But Canary: Configuration Safety at Scale </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-17484 type-post status-publish format-standard has-post-thumbnail hentry category-developer-tools fb_content_type-article" id="post-23840">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/04/Compass-Not-Enclycopedia-Hero.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-04-06T09:00:50-07:00">
APR 6, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/04/06/developer-tools/how-meta-used-ai-to-map-tribal-knowledge-in-large-scale-data-pipelines/" rel="bookmark" aria-label="Read More about How Meta Used AI to Map Tribal Knowledge in Large-Scale Data Pipelines">
How Meta Used AI to Map Tribal Knowledge in Large-Scale Data Pipelines </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
</div><!-- /.row -->
</div><!--/.article-grids -->
</div><!-- /.related-articles -->
<!-- include the carousel nav here. -->
</main><!-- #main -->
</div><!-- #primary -->
<!-- Do the right sidebar check. -->
<div class="col-md-4 widget-area" id="right-sidebar" role="complementary">
<aside id="fb_related_post_widget-4" class="widget widget_fb_related_post_widget"><h3 class="widget-title">Related Posts</h3><hr>
<div class="aside-related-projects"><div class="article-grid-redesign">
<article class="post-17271 post type-post status-publish has-post-thumbnail hentry" id="post-17271">
<a href="https://engineering.fb.com/2021/02/17/developer-tools/fix-fast/" rel="bookmark" aria-label="Read More about Faster, more efficient systems for finding and fixing regressions">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_HERO_FINAL.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_HERO_FINAL.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2021/02/FixFast_HERO_FINAL.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Feb 17, 2021">Feb 17, 2021</time></span>
<h4 class="entry-title">Faster, more efficient systems for finding and fixing regressions</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div><div class="article-grid-redesign">
<article class="post-17058 post type-post status-publish has-post-thumbnail hentry" id="post-17058">
<a href="https://engineering.fb.com/2020/12/10/developer-tools/probabilistic-flakiness/" rel="bookmark" aria-label="Read More about How do you test your tests?">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2020/12/Rsys_Hero.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2020/12/Rsys_Hero.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2020/12/Rsys_Hero.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Dec 10, 2020">Dec 10, 2020</time></span>
<h4 class="entry-title">How do you test your tests?</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div><div class="article-grid-redesign">
<article class="post-17180 post type-post status-publish has-post-thumbnail hentry" id="post-17180">
<a href="https://engineering.fb.com/2021/02/09/developer-tools/minesweeper/" rel="bookmark" aria-label="Read More about Minesweeper automates root cause analysis as a first-line defense against bugs">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2021/02/Minesweeper-Hero.png?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2021/02/Minesweeper-Hero.png?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2021/02/Minesweeper-Hero.png?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Feb 09, 2021">Feb 09, 2021</time></span>
<h4 class="entry-title">Minesweeper automates root cause analysis as a first-line defense against bugs</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div></div></aside><aside id="wpb_widget-3" class="widget widget_wpb_widget"><h3 class="widget-title">Related Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2316600655751692/>
<div class="job-description">
Machine Learning Engineer
</div>
<div class="job-location">
TEL AVIV, IL
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2887906508264541/>
<div class="job-description">
Software Engineer (Leadership) - Product
</div>
<div class="job-location">
LONDON, GB
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/1645503493182815/>
<div class="job-description">
Software Engineer, Machine Learning RecSys
</div>
<div class="job-location">
SUNNYVALE, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/1645503493182815/>
<div class="job-description">
Software Engineer, Machine Learning RecSys
</div>
<div class="job-location">
BELLEVUE, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/1645503493182815/>
<div class="job-description">
Software Engineer, Machine Learning RecSys
</div>
<div class="job-location">
MENLO PARK, US
</div>
</a>
</li> </ul>
<a href="https://www.metacareers.com">
<div class="btn available-positions-btn">
See All Jobs </div>
</a>
</aside>
</div><!-- #secondary -->
<!-- <php get_template_part('content-single-side-bar'); ?> -->
<!-- Do the right sidebar check. -->
</div><!-- Container end -->
</div><!-- Wrapper end -->
</div>
<!-- ******************* The Footer Full-width Widget Area ******************* -->
<div class="wrapper wrapper-community-outer">
<div class="wrapper-community">
<div class="community" tabindex="-1">
<div class="row">
<!-- community section is generated through WordPress widget -->
<div id="wpb_widget-2" class="footer-widget widget_wpb_widget widget-count-3 col-md-4"><h3 class="widget-title">Available Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2316600655751692/>
<div class="job-description">
Machine Learning Engineer
</div>
<div class="job-location">
TEL AVIV, IL
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2887906508264541/>
<div class="job-description">
Software Engineer (Leadership) - Product
</div>
<div class="job-location">
LONDON, GB
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/1645503493182815/>
<div class="job-description">
Software Engineer, Machine Learning RecSys
</div>
<div class="job-location">
SUNNYVALE, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/1645503493182815/>
<div class="job-description">
Software Engineer, Machine Learning RecSys
</div>
<div class="job-location">
BELLEVUE, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/1645503493182815/>
<div class="job-description">
Software Engineer, Machine Learning RecSys
</div>
<div class="job-location">
MENLO PARK, US
</div>
</a>
</li> </ul>
<a href="https://www.metacareers.com">
<div class="btn available-positions-btn">
See All Jobs </div>
</a>
</div><!-- .footer-widget --><div id="custom_html-2" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Technology at Meta</h3><div class="textwidget custom-html-widget"><ul class="stay-connected-lists">
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-fb-engineering" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Engineering at Meta - X
</div>
<a onclick='return twitterFollow("Meta_Engineers")'>
<button class="btn stay-connected-btn">
Follow
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-AI" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
AI at Meta
</div>
<a href="https://ai.meta.com/blog/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta Quest Blog
</div>
<a href="https://www.meta.com/blog/quest/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta for Developers
</div>
<a href="https://developers.facebook.com/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-bug-bounty" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta Bug Bounty
</div>
<a href="https://bugbounty.meta.com/">
<button class="btn stay-connected-btn">
Learn more
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/rss.png" alt="footer-rss" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
RSS
</div>
<a href="https://code.facebook.com/posts/rss/">
<button class="btn stay-connected-btn">
Subscribe
</button>
</a>
</div>
</li>
</ul>
</div></div><!-- .footer-widget --><div id="custom_html-3" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Open Source</h3><div class="textwidget custom-html-widget"><p class="opensource-desc">
Meta believes in building community through open source technology. Explore our latest projects in Artificial Intelligence, Data Infrastructure, Development Tools, Front End, Languages, Platforms, Security, Virtual Reality, and more.
</p>
<ul class="opensource-cats">
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/android.png" alt="android" loading="lazy" width="50" height="59">
<div class="opensource-cap">
ANDROID
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/ios.png" alt="ios" loading="lazy" width="48" height="58">
<div class="opensource-cap">
iOS
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/web.png" alt="web" loading="lazy" width="55" height="49">
<div class="opensource-cap">
WEB
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/backend.png" alt="backend" loading="lazy" width="56" height="51">
<div class="opensource-cap">
BACKEND
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/hardware.png" alt="hardware" loading="lazy" width="55" height="44">
<div class="opensource-cap">
HARDWARE
</div>
</li>
</ul>
<button class="btn opensource-btn" onclick="window.location.href='https://opensource.fb.com/'">
Learn More
</button>
</div></div><!-- .footer-widget -->
</div>
</div>
</div>
</div><!-- #wrapper-footer-full -->
<div class="wrapper-footer" id="wrapper-footer">
<div class="inner-container">
<div class="footer">
<div class="row">
<div class="col-md-12">
<footer class="meta-footer" id="meta-footer">
<div class="site-info row justify-content-between">
<div class="col-md-5 meta-logo-container">
<a href="https://about.facebook.com/">
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/meta_logo_full.svg" width="89" height="18" alt="Meta" />
</a>
<div id="text-2" class="widget widget_text"> <div class="textwidget"><p>Engineering at Meta is a technical news resource for engineers interested in how we solve large-scale technical challenges at Meta.</p>
</div>
</div> </div>
<div class="col-md-6 footer-meta-links-container">
<ul class="meta-footer-links">
<li class="footer-link"><a href="https://engineering.fb.com">Home</a></li><li class="footer-link"><a href="https://about.meta.com/">Company Info</a></li><li class="footer-link"><a href="https://www.metacareers.com/?ref=engineering.fb.com">Careers</a></li> </ul>
</div>
</div>
</footer>
</div>
</div>
</div>
</div>
<div class="footer">
<div class="row">
<div class="col-md-12">
<footer class="site-footer" id="colophon">
<div class="inner-container">
<div class="site-info row justify-content-between">
<div class="col-md-4 copyright-container">
<div class="copyright"> © 2026 Meta</div>
</div>
<!-- static footer -->
<div class="col-md-7 footer-links-container">
<ul class="footer-links">
<li class="footer-link"><a href="https://www.facebook.com/policies">Terms</a></li><li class="footer-link"><a href="https://www.facebook.com/privacy/policy">Privacy</a></li><li class="footer-link"><a href="/privacy">Cookies</a></li><li class="footer-link"><a href="https://www.facebook.com/help">Help</a></li> </ul>
</div>
</div>
</div><!-- .site-info -->
</footer><!-- #colophon -->
</div><!--col end -->
</div><!-- row end -->
</div><!-- //.inner-container -->
</div><!-- wrapper end -->
</div><!-- #page we need this extra closing tag here -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/code-fb-com/*","/wp-content/themes/understrap/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<div class="gdprconsent-container" id="GDPRConsentBar">
<div class="gdprconsent-wrapper">
<div class="gdprconsent-content">
<p>To help personalize content, tailor and measure ads and provide a safer experience, we use cookies. By clicking or navigating the site, you agree to allow our collection of information on and off Facebook through cookies. Learn more, including about available controls: <a href="/privacy">Cookie Policy</a></p>
<div class="gdprconsent-button">
<button type="button" data-gdpr-accept>Accept</button>
</div>
</div>
</div>
</div>
<!-- Twitter universal website tag code -->
<script>
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
// Insert Twitter Pixel ID and Standard Event data below
twq('init','o31u0');
twq('track','PageView');
});
}
</script>
<!-- End Twitter universal website tag code -->
<script type="text/javascript">
window.WPCOM_sharing_counts = {"https://engineering.fb.com/2021/04/27/developer-tools/reverse-debugging/":17484};
</script>
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KZRLGZ8" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=660742068067366&ev=PageView&noscript=1"/></noscript>
<script id="child-understrap-scripts-js-extra">
var loadmore_params = {"restfulURL":"https://engineering.fb.com/wp-json/fb/v1/loadmore","searchText":"","posts":"{\"page\":0,\"year\":2021,\"monthnum\":4,\"day\":27,\"name\":\"reverse-debugging\",\"category_name\":\"developer-tools\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","post_type":"post"};
var search_params = {"ajaxurl":"https://engineering.fb.com/wp-admin/admin-ajax.php","posts":"{\"page\":0,\"year\":2021,\"monthnum\":4,\"day\":27,\"name\":\"reverse-debugging\",\"category_name\":\"developer-tools\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","article_filter":"2f23ad5ce3"};
var more_articles = {"next_article":{"label":"Next","permalink":"https://engineering.fb.com/2021/04/29/developer-tools/rust/","title":"A brief history of Rust at Facebook","thumbnail":["https://engineering.fb.com/wp-content/uploads/2021/04/Rust-Facebook-Hero.jpeg?w=580&h=326&crop=1",580,326,true]},"prev_article":{"label":"Prev","permalink":"https://engineering.fb.com/2021/04/16/security/dit/","title":"DIT \u2014 enabling de-identified data collection on WhatsApp","thumbnail":["https://engineering.fb.com/wp-content/uploads/2021/04/DIT-1.png?w=580&h=326&crop=1",580,326,true]}};
//# sourceURL=child-understrap-scripts-js-extra
</script>
<script type="text/javascript" src="https://engineering.fb.com/_static/??/wp-content/themes/code-fb-com/dist/js/child-theme.min.js,/wp-content/themes/code-fb-com/js/prism.js?m=1783017802j" ></script><script id="sharing-js-js-extra">
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":""};
//# sourceURL=sharing-js-js-extra
</script>
<script id="sharing-js-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/_inc/build/sharedaddy/sharing.min.js?ver=16.1.3"></script>
<script id="sharing-js-js-after">
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-facebook' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-threads' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-threads' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomthreads', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-linkedin' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-x' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-x' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomx', 'menubar=1,resizable=1,width=600,height=350' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-bluesky' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-bluesky' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcombluesky', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-mastodon' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-mastodon' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcommastodon', 'menubar=1,resizable=1,width=460,height=400' );
return false;
}
} );
} )();
//# sourceURL=sharing-js-js-after
</script>
<script id="video-analytics-js-after">
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
let youtube = document.querySelectorAll('[src*="youtube-nocookie"]');
for (let i = 0; i < youtube.length; i++) {
youtube[i].src = youtube[i].src.replace('youtube-nocookie.com', 'youtube.com');
}
let vimeo = document.querySelectorAll('[src*="vimeo"]');
for (let i = 0; i < vimeo.length; i++) {
vimeo[i].src = vimeo[i].src.replace('&dnt=true', '');
}
});
}
//# sourceURL=video-analytics-js-after
</script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://engineering.fb.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://engineering.fb.com/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>