Files
nexus/sreweekly/articles/212/03-ntp-building-a-more-accurate-time-service-at-facebook-scale.html
2026-09-12 17:23:01 +08:00

1397 lines
132 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-title" content="Engineering at Meta - Engineering at Meta Blog">
<meta property="fb:app_id" content="1425766027653270" />
<link rel="profile" href="http://gmpg.org/xfn/11">
<link rel="pingback" href="https://engineering.fb.com/xmlrpc.php">
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
<script async src="https://engineering.fb.com/wp-content/themes/code-fb-com/dist/js/optimize.min.js"></script>
<!-- This site is optimized with the Yoast SEO Premium plugin v19.3 (Yoast SEO v27.3) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
<title>NTP: Building a more accurate time service at Facebook scale</title>
<meta name="description" content="We&#039;ve improved time precision within our infrastructure from 10 milliseconds to 100 microseconds, and released our own public NTP service." />
<link rel="canonical" href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/" />
<meta name="twitter:label1" content="Written by" />
<meta name="twitter:data1" content="Oleg Obleukhov" />
<meta name="twitter:label2" content="Est. reading time" />
<meta name="twitter:data2" content="14 minutes" />
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/#article","isPartOf":{"@id":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/"},"author":[{"@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/5c4b775040a892703d52f0386b4ed13d"}],"headline":"Building a more accurate time service at Facebook scale","datePublished":"2020-03-18T12:00:39+00:00","dateModified":"2024-03-25T16:00:17+00:00","mainEntityOfPage":{"@id":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/"},"wordCount":2525,"publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"image":{"@id":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2020\/03\/Time-infra-hero-image.jpg","articleSection":["Production Engineering"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/","url":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/","name":"NTP: Building a more accurate time service at Facebook scale","isPartOf":{"@id":"https:\/\/engineering.fb.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/#primaryimage"},"image":{"@id":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/#primaryimage"},"thumbnailUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2020\/03\/Time-infra-hero-image.jpg","datePublished":"2020-03-18T12:00:39+00:00","dateModified":"2024-03-25T16:00:17+00:00","description":"We've improved time precision within our infrastructure from 10 milliseconds to 100 microseconds, and released our own public NTP service.","breadcrumb":{"@id":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/#primaryimage","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2020\/03\/Time-infra-hero-image.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2020\/03\/Time-infra-hero-image.jpg","width":2000,"height":1125,"caption":"Building a more accurate time service at Facebook scale"},{"@type":"BreadcrumbList","@id":"https:\/\/engineering.fb.com\/2020\/03\/18\/production-engineering\/ntp-service\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/engineering.fb.com\/"},{"@type":"ListItem","position":2,"name":"Building a more accurate time service at Facebook scale"}]},{"@type":"WebSite","@id":"https:\/\/engineering.fb.com\/#website","url":"https:\/\/engineering.fb.com\/","name":"Engineering at Meta","description":"Engineering at Meta Blog","publisher":{"@id":"https:\/\/engineering.fb.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/engineering.fb.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/engineering.fb.com\/#organization","name":"Meta","url":"https:\/\/engineering.fb.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2023\/08\/Meta_lockup_positive-primary_RGB.jpg","width":29011,"height":12501,"caption":"Meta"},"image":{"@id":"https:\/\/engineering.fb.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/Engineering\/","https:\/\/x.com\/fb_engineering"]},[],{"@type":"Person","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/5c4b775040a892703d52f0386b4ed13d","name":"Oleg Obleukhov","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/engineering.fb.com\/#\/schema\/person\/image\/d59408ca3e16a8608b7228e1f136036b","url":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2020\/03\/olego.jpg?w=200&h=200&crop=1","contentUrl":"https:\/\/engineering.fb.com\/wp-content\/uploads\/2020\/03\/olego.jpg?w=200&h=200&crop=1","width":200,"height":200,"caption":"Oleg Obleukhov"},"description":"Production Engineer","url":"https:\/\/engineering.fb.com\/author\/oleg-obleukhov\/"}]}</script>
<!-- / Yoast SEO Premium plugin. -->
<link rel='dns-prefetch' href='//secure.gravatar.com' />
<link rel='dns-prefetch' href='//v0.wordpress.com' />
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta &raquo; Feed" href="https://engineering.fb.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="Engineering at Meta &raquo; Comments Feed" href="https://engineering.fb.com/comments/feed/" />
<script async src="https://www.googleoptimize.com/optimize.js?id=OPT-TPNDQC5" onerror="dataLayer.hide.end && dataLayer.hide.end()"></script>
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2020%2F03%2F18%2Fproduction-engineering%2Fntp-service%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.fb.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.fb.com%2F2020%2F03%2F18%2Fproduction-engineering%2Fntp-service%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='all-css-12' href='https://engineering.fb.com/_static/??-eJyNjUsOwjAMRC9EaiqghQXiKFVqhzSQn+pEiNuTFJBAbNjZTzNv4BYFBp+UTxBt1sYzFKQpzpVz5WQ4QSUv0CDzCj6KaVJOMWAgJc5jwe7ZKTnAyVgSS6Jxxv90XRbv3YtKUeJVtF3TgguUbZHyJGdFkui+nMbrvxWD8QhjLvvAAY20wgYd+OupspM7tv3+sO533Wb7AImQY4U=' type='text/css' media='all' />
<script id="gdprconsent-js" src="https://engineering.fb.com/wp-content/plugins/wp-gdpr-consent/dist/gdprconsent.js?ver=2.0.0"></script>
<script id="gdprconsent-js-after">
if (window.gdprSafeTrack) {
window.gdprSafeTrack('https://www.googletagmanager.com/gtag/js?id=391063774');
window.gdprSafeTrack(function() {
var google_analytics_g4 = decodeURIComponent( '391063774' );
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', google_analytics_g4);
});
}
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
var google_tag_manager = decodeURIComponent( 'GTM-KZRLGZ8' );
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer',google_tag_manager);
});
}
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
var facebook_pixel = decodeURIComponent( '660742068067366' );
!function(f,b,e,v,n,t,s)
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
n.queue=[];t=b.createElement(e);t.async=!0;
t.src=v;s=b.getElementsByTagName(e)[0];
s.parentNode.insertBefore(t,s)}(window, document,'script',
'https://connect.facebook.net/en_US/fbevents.js');
fbq('init', facebook_pixel);
fbq('track', 'PageView');
});
}
//# sourceURL=gdprconsent-js-after
</script>
<script type="text/javascript" src="https://engineering.fb.com/_static/??-eJzTLy/QzcxLzilNSS3WzwKiwtLUokoopZebmaeXVayjj0+Rbm5melFiSSpUsX2uraG5hbmZgYmloUkWAK+3Iic=" ></script><script type="text/javascript" src="https://engineering.fb.com/wp-content/themes/understrap/js/popper.min.js?m=1773259670g" ></script><link rel="https://api.w.org/" href="https://engineering.fb.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.fb.com/wp-json/wp/v2/posts/14157" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.fb.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress 7.1" />
<link rel='shortlink' href='https://wp.me/pa0Lhq-3Gl' />
<link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/FacebookReader-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://engineering.fb.com/wp-content/themes/code-fb-com/fonts/Optimistic_Display_W_Md.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="Building a more accurate time service at Facebook scale" />
<meta property="og:url" content="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/" />
<meta property="og:description" content="UPDATE: To continue our support of this public NTP service, we have open-sourced our collection of NTP libraries on GitHub. Almost all of the billions of devices connected to the internet have onbo…" />
<meta property="article:published_time" content="2020-03-18T12:00:39+00:00" />
<meta property="article:modified_time" content="2024-03-25T16:00:17+00:00" />
<meta property="og:site_name" content="Engineering at Meta" />
<meta property="og:image" content="https://engineering.fb.com/wp-content/uploads/2020/03/Time-infra-hero-image.jpg" />
<meta property="og:image:width" content="2000" />
<meta property="og:image:height" content="1125" />
<meta property="og:image:alt" content="Building a more accurate time service at Facebook scale" />
<meta property="og:locale" content="en_US" />
<meta name="twitter:text:title" content="Building a more accurate time service at Facebook scale" />
<meta name="twitter:image" content="https://engineering.fb.com/wp-content/uploads/2020/03/Time-infra-hero-image.jpg?w=640" />
<meta name="twitter:image:alt" content="Building a more accurate time service at Facebook scale" />
<meta name="twitter:card" content="summary_large_image" />
<!-- End Jetpack Open Graph Tags -->
<style id="wp-custom-css">
.table-of-contents {
position: sticky;
top: 20px;
background: #fff;
padding: 1em;
border: 1px solid #ddd;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
max-width: 250px;
}
</style>
<link rel="shortcut icon" href="https://engineering.fb.com/wp-content/themes/code-fb-com/favicon.ico" />
</head>
<body class="wp-singular post-template-default single single-post postid-14157 single-format-standard wp-theme-understrap wp-child-theme-code-fb-com group-blog">
<div class="hfeed site" id="page">
<!-- ******************* The Navbar Area ******************* -->
<div class="wrapper-fluid wrapper-navbar" id="wrapper-navbar">
<a class="skip-link screen-reader-text sr-only" href="#content">
Skip to content </a>
<nav class="navbar navbar-expand-std">
<div class="nav-container">
<!-- Your site title as branding in the menu -->
<div class="nav-top-row">
<h1 class="navbar-brand mb-0">
<a rel="home" href="https://engineering.fb.com/" title="Engineering at Meta">
<img class="navbar-brand-logo" src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/logo-meta.svg" class="navbar-brand-logo" alt="Engineering at Meta" width="215" height="40" loading="lazy">
</a>
</h1>
<!-- end custom logo -->
<label for="search" class="screen-reader-text">Search this site</label>
<input type="text" id="search" name="search" class="form-control-lg navbar-search" autocomplete="off">
<button class="navbar-toggler navbar-searchbar-toggler"
type="button"
data-toggle="collapse"
data-target="#searchbarDropdown"
aria-controls="searchbarDropdown"
aria-expanded="false"
aria-label="Toggle navigation">
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/icon-search.svg" class="navbar-mobile-search-icon" loading="lazy" width="24" height="25" />
</button>
<!-- desktop language menu (mobile language menu is implemented via jQuery)-->
<div class="notranslate nice-select navbar-language form-control">
</div>
<button class="navbar-toggler collapsed navbar-menu-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<div class="navbar-border"></div>
<!-- The WordPress Menu goes here -->
<div id="navbarNavDropdown" class="collapse navbar-collapse"><ul id="main-menu" class="navbar-nav"><li id="menu-item-60" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-60 dropdown"><a title="Open Source" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Open Source <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-12330" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-12330"><a title="Open Source" href="https://engineering.fb.com/category/open-source/" class="nav-link">Open Source</a></li>
<li id="menu-item-12331" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-12331"><a title="Meta Open Source" href="https://opensource.fb.com" class="nav-link">Meta Open Source</a></li>
</ul>
</li>
<li id="menu-item-2686" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2686 dropdown"><a title="Platforms" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Platforms <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-65" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-65"><a title="Android" href="https://engineering.fb.com/category/android/" class="nav-link">Android</a></li>
<li id="menu-item-66" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-66"><a title="iOS" href="https://engineering.fb.com/category/ios/" class="nav-link">iOS</a></li>
<li id="menu-item-67" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-67"><a title="Web" href="https://engineering.fb.com/category/web/" class="nav-link">Web</a></li>
</ul>
</li>
<li id="menu-item-2682" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2682 dropdown"><a title="Infrastructure Systems" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Infrastructure Systems <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-218" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-218"><a title="Core Infra" href="https://engineering.fb.com/category/core-infra/" class="nav-link">Core Infra</a></li>
<li id="menu-item-219" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-219"><a title="Data Infrastructure" href="https://engineering.fb.com/category/data-infrastructure/" class="nav-link">Data Infrastructure</a></li>
<li id="menu-item-220" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-220"><a title="DevInfra" href="https://engineering.fb.com/category/developer-tools/" class="nav-link">DevInfra</a></li>
<li id="menu-item-226" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent nav-item menu-item-226"><a title="Production Engineering" href="https://engineering.fb.com/category/production-engineering/" class="nav-link">Production Engineering</a></li>
<li id="menu-item-227" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-227"><a title="Security &amp; Privacy" href="https://engineering.fb.com/category/security/" class="nav-link">Security &amp; Privacy</a></li>
<li id="menu-item-18607" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18607"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/systems-infrastructure/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2683" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2683 dropdown"><a title="Physical Infrastructure" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Physical Infrastructure <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-215" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-215"><a title="Connectivity" href="https://engineering.fb.com/category/connectivity/" class="nav-link">Connectivity</a></li>
<li id="menu-item-216" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-216"><a title="Data Center Engineering" href="https://engineering.fb.com/category/data-center-engineering/" class="nav-link">Data Center Engineering</a></li>
<li id="menu-item-217" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-217"><a title="Networking &amp; Traffic" href="https://engineering.fb.com/category/networking-traffic/" class="nav-link">Networking &amp; Traffic</a></li>
<li id="menu-item-18608" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-18608"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/networking-connectivity/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2684" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2684 dropdown"><a title="Video Engineering &amp; AR/VR" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Video Engineering &amp; AR/VR <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-225" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-225"><a title="Video Engineering" href="https://engineering.fb.com/category/video-engineering/" class="nav-link">Video Engineering</a></li>
<li id="menu-item-223" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-223"><a title="Virtual Reality" href="https://engineering.fb.com/category/virtual-reality/" class="nav-link">Virtual Reality</a></li>
<li id="menu-item-13388" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13388"><a title="Research Publications" href="https://research.facebook.com/publications/research-areas/augmented-reality-virtual-reality/" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-2685" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-has-children nav-item menu-item-2685 dropdown"><a title="Artificial Intelligence" href="#" data-toggle="dropdown" class="nav-link dropdown-toggle">Artificial Intelligence <span class="caret"></span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 129 129" enable-background="new 0 0 129 129" width="12px" height="15px">
<polyline points="40 50 80 90 120 50" stroke="#adb1b8" stroke-width="10" stroke-linecap="butt" fill="none" stroke-linejoin="miter"/>
</svg></a>
<ul class=" dropdown-menu" role="menu">
<li id="menu-item-222" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-222"><a title="ML Applications" href="https://engineering.fb.com/category/ml-applications/" class="nav-link">ML Applications</a></li>
<li id="menu-item-221" class="menu-item menu-item-type-taxonomy menu-item-object-category nav-item menu-item-221"><a title="AI Research" href="https://engineering.fb.com/category/ai-research/" class="nav-link">AI Research</a></li>
<li id="menu-item-13387" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-13387"><a title="Research Publications" href="https://ai.facebook.com/results/?content_types%5B0%5D=publication" class="nav-link">Research Publications</a></li>
</ul>
</li>
<li id="menu-item-228" class="menu-item menu-item-type-custom menu-item-object-custom nav-item menu-item-228"><a title="Watch Videos" href="/videos" class="nav-link">Watch Videos</a></li>
</ul></div> </div><!-- .container -->
<div class="search-container collapse" id="searchbarDropdown">
<div class="searchbar-search">
<div class="searchbar-container">
<div class="navbar-mobile-search-margin"></div>
<input type="text" name="search" placeholder="Search" class="form-control-lg navbar-mobile-search" autofocus autocomplete="off">
<button class="searchbar-clear">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
</div>
</div>
</nav><!-- .site-navigation -->
</div><!-- .wrapper-navbar end -->
<div class="wrapper" id="single-wrapper">
<div class="single-container row" id="content" tabindex="-1">
<!-- Do the left sidebar check. -->
<div class="content-area" id="primary">
<main class="site-main" id="main">
<article class="post-14157 post type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-14157">
<header class="entry-header">
<div class="entry-meta">
POSTED ON <span class="posted-on entry-date"><time class="published updated" datetime="2020-03-18">MARCH 18, 2020</time></span> TO <span class="cat-links"><a href="https://engineering.fb.com/category/production-engineering/" rel="category tag">Production Engineering</a></span>
<h1 class="entry-title"><div>Building a more accurate time service at Facebook scale</div></h1> </div><!-- .entry-meta -->
</header><!-- .entry-header -->
<figure id="post-feat-image-container">
<img width="2000" height="1125" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time-infra-hero-image.jpg" class="wp-post-image" alt="Building a more accurate time service at Facebook scale" title="Time_infra_hero" /> </figure>
<div class="entry-authors-social">
<div class="entry-authors">
By <a href="https://engineering.fb.com/author/oleg-obleukhov/" title="Posts by Oleg Obleukhov" class="author url fn" rel="author">Oleg Obleukhov</a> </div>
</div>
<div class="entry-content">
<p><span style="font-weight: 400;"><strong>UPDATE:</strong> To continue our support of this public NTP service, we have <a href="https://github.com/facebookincubator/ntp">open-sourced our collection of NTP libraries on GitHub</a>.</span></p>
<p><span style="font-weight: 400;">Almost all of the billions of devices connected to the internet have onboard clocks, which need to be accurate to properly perform their functions. Many clocks contain inaccurate internal oscillators, which can cause seconds of inaccuracy per day and need to be periodically corrected. Incorrect time can lead to issues, such as missing an important reminder or failing a spacecraft launch. Devices all over the world rely on Network Time Protocol (NTP) to stay synchronized to a more accurate clock over packet-switched, variable-latency data networks.</span></p>
<p><span style="font-weight: 400;">As Facebook’s infrastructure has grown, time precision in our systems has become more and more important. We need to know the accurate time difference between two random servers in a data center so that datastore writes don’t mix up the order of transactions. We need to sync all the servers across many data centers with sub-millisecond precision. For that we tested</span><a href="https://chrony.tuxfamily.org/"> <span style="font-weight: 400;">chrony</span></a><span style="font-weight: 400;">, a modern NTP server implementation with interesting features. During testing, we found that chrony is significantly more accurate and scalable than the previously used service, ntpd, which made it an easy decision for us to replace ntpd in our infrastructure. Chrony also forms the foundation of our Facebook public NTP service, available from time.facebook.com. In this post, we will share our work to improve accuracy from 10 milliseconds to 100 microseconds and how we verified these results in our timing laboratory.</span></p>
<h2><span style="font-weight: 400;">Leap second</span></h2>
<p><span style="font-weight: 400;">Before we dive into the details of our NTP service, we need to look at a phenomenon called a leap second. Because of the Earth’s rotation irregularities, we occasionally need to add or remove a second from time, or a </span><a href="https://caps.gsfc.nasa.gov/simpson/time/leapseconds.html"><span style="font-weight: 400;">leap second</span></a><span style="font-weight: 400;">. For humans, adding or removing a second creates an almost unnoticeable hiccup when watching a clock. Servers, however, can miss a ton of transactions or events or experience a serious software malfunction when they expect time to go forward continuously. One of the most popular approaches for addressing that is to “</span><a href="https://docs.ntpsec.org/latest/leapsmear.html"><span style="font-weight: 400;">smear” the leap second</span></a><span style="font-weight: 400;">, which means to change the time in very small increments across multiple hours</span><span style="font-weight: 400;">.</span></p>
<h2><span style="font-weight: 400;">Building an NTP service at scale</span></h2>
<p><span style="font-weight: 400;">Facebook NTP service is designed in four layers, or strata: </span></p>
<ul>
<li style="font-weight: 400;"><span style="font-weight: 400;">Stratum 0 is a layer of satellites with extremely precise atomic clocks from a global navigation satellite system (GNSS), such as GPS, </span><a href="https://gssc.esa.int/navipedia/index.php/GLONASS_General_Introduction"><span style="font-weight: 400;">GLONASS</span></a><span style="font-weight: 400;">, or </span><a href="https://galileognss.eu/"><span style="font-weight: 400;">Galileo</span></a><span style="font-weight: 400;">.</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">Stratum 1 is Facebook atomic clock synchronizing with a GNSS.</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">Stratum 2 is a pool of NTP servers synchronizing to Stratum 1 devices. Leap-second smearing is happening at this stage.</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">Stratum 3 is a tier of servers configured for a larger scale. They receive smeared time and are ignorant of leap seconds.</span></li>
</ul>
<p><span style="font-weight: 400;">There could be as many as 16 layers to distribute the load in some systems. The number of layers depends on the scale and precision requirements. </span></p>
<p><img decoding="async" class="alignnone wp-image-14160 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-NTP-Service.jpg?w=916" alt="Facebook's NTP service is designed in four layers, or strata: " width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-NTP-Service.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-NTP-Service.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-NTP-Service.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-NTP-Service.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-NTP-Service.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-NTP-Service.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-NTP-Service.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-NTP-Service.jpg?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">When we set out to build our NTP service, we tested the following time daemons to use:</span></p>
<ol>
<li style="font-weight: 400;"><b>Ntpd: </b><span style="font-weight: 400;">A common daemon, ntpd used to be included on most Unix-like operating systems. It has been a stable solution for many years and may be running on your computer right now.</span></li>
<li style="font-weight: 400;"><b>Chrony: </b><span style="font-weight: 400;">A fairly new daemon, chrony has interesting features and the potential to provide more precise time synchronization using NTP. Chrony also offers its own extended control protocol and theoretically could bring precision down to nanoseconds. From a resource consumption perspective, we found ntpd and chrony to be fairly similar, though chrony seems to consume slightly less RAM (~1 MiB difference). </span></li>
</ol>
<p><img decoding="async" class="alignnone wp-image-14161 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-CPU-Time.jpg?w=916" alt="From a resource consumption perspective, we found ntpd and chrony to be fairly similar, though chrony seems to consume slightly less RAM (~1 MiB difference). " width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-CPU-Time.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-CPU-Time.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-CPU-Time.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-CPU-Time.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-CPU-Time.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-CPU-Time.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-CPU-Time.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-CPU-Time.jpg?resize=192,108 192w" sizes="(max-width: 992px) 100vw, 62vw" /></p>
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-14185" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-02.jpg" alt="From a resource consumption perspective, we found ntpd and chrony to be fairly similar, though chrony seems to consume slightly less RAM (~1 MiB difference). " width="2000" height="1125" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-02.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-02.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-02.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-02.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-02.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-02.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-02.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-02.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<h2><span style="font-weight: 400;">Daemon estimates</span></h2>
<p><span style="font-weight: 400;">Whether a system is using ntpd or chrony, each provides some estimated measurements. The daemon includes command line tools that display these estimates, as shown below. These estimates rely on many assumptions, such as:</span></p>
<ul>
<li style="font-weight: 400;"><span style="font-weight: 400;">The network path between client and server is symmetric.</span></li>
<li style="font-weight: 400;">When timestamps are added to the NTP packet and <code>send()</code> is called, the operating system will dispatch it immediately.</li>
<li style="font-weight: 400;">Oscillator temperature and supplied voltage are constant.</li>
</ul>
<p><span style="font-weight: 400;">Ntpd includes ntpq command line tool, which displays how out of sync it determines that time is:</span></p>
<pre class="line-numbers"><code class="language-javascript">[user@client ~]# ntpq -p
remote refid st t when poll reach delay offset jitter
=========================================================================
+server1 .FB. 2 u 406 1024 377 0.029 0.149 0.035
+server2 .FB. 2 u 548 1024 377 0.078 0.035 0.083
*server3 .FB. 2 u 460 1024 377 0.049 -0.185 0.114</code></pre>
<p><span style="font-weight: 400;">However, can we trust these numbers? If ntpd reports that the time is off by 0.185 ms, is that accurate? The short answer is no. Server estimates the offset based on multiple timestamps in a packet. And the actual values are supposed to be within a 10x bigger window. In other words, a result saying time is off by 0.185 ms means it’s probably within +/-2 ms (so, 4 ms in total). However, our testing demonstrated that accuracy is generally within 10 ms with ntpd.</span></p>
<p><span style="font-weight: 400;">We had technical requirements for better precision. For example, multi-master databases translate microseconds and even nanoseconds of precision directly into their theoretical throughput. Another example for which moderate precision is required is logging — in order to match logs between nodes of a distributed system, milliseconds of precision is often required.</span></p>
<p><span style="font-weight: 400;">Let’s see what happens if we replace ntpd with chrony:</span></p>
<pre class="line-numbers"><code class="language-javascript">[user@client ~]# chronyc sources -v
210 Number of sources = 19
.-- Source mode '^' = server, '=' = peer, '#' = local clock.
/ .- Source state '*' = current synced, '+' = combined , '-' = not combined,
| / '?' = unreachable, 'x' = time may be in error, '~' = time too variable.
|| .- xxxx [ yyyy ] +/- zzzz
|| Reachability register (octal) -. | xxxx = adjusted offset,
|| Log2(Polling interval) --. | | yyyy = measured offset,
|| \ | | zzzz = estimated error.
|| | | \
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
^+ server1 2 6 377 2 +10us[ +10us] +/- 481us
^- server2 2 6 377 0 -4804us[-4804us] +/- 77ms
^* server3 2 6 377 59 -42us[ -46us] +/- 312us
^+ server4 3 6 377 60 +11ns[-3913ns] +/- 193us
</code></pre>
<p><span style="font-weight: 400;">Pay attention to those last three numbers. Backwards, from right to left:</span></p>
<ul>
<li style="font-weight: 400;"><span style="font-weight: 400;">The very last number is the estimated error. It’s prefixed with +/-. This is how much chrony evaluates the maximum error. Sometimes it’s 10s of milliseconds, sometimes it’s 100s of microseconds (100x difference). This is because when chrony synchronizes with another chrony, it uses the </span><a href="https://tools.ietf.org/html/draft-ietf-ntp-interleaved-modes-01"><span style="font-weight: 400;">extended NTP protocol</span></a><span style="font-weight: 400;">, which dramatically increases precision. Not too bad!</span></li>
</ul>
<ul>
<li style="font-weight: 400;"><span style="font-weight: 400;">Next is the number in the square brackets. It shows the measured offset. We can see approximately 100x difference here as well, except for server4 (we will talk about this one later).</span></li>
</ul>
<ul>
<li style="font-weight: 400;"><span style="font-weight: 400;">The number to the left of the square brackets shows the original measurement, adjusted to allow for any slews applied to the local clock since that first measurement. Again, we can see 100x difference between ntpd and chrony.</span></li>
</ul>
<p><span style="font-weight: 400;">Let’s see how it looks in the graph:</span><span style="font-weight: 400;"><br />
</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14198 size-full" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-03-1.jpg" alt="The vertical blue line represents the moment when ntpd was replaced with chrony. With ntpd, we were in the range of +/-1.5 ms. With chrony, we are in the range of microseconds. " width="2000" height="1125" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-03-1.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-03-1.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-03-1.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-03-1.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-03-1.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-03-1.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-03-1.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-03-1.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">The vertical blue line represents the moment when ntpd was replaced with chrony. With ntpd, we were in the range of +/-1.5 ms. With chrony, we are in the range of microseconds. More importantly, estimated error (window) dropped to a range of 100s of microseconds, which we can confirm with lab measurements (more on this below). However, these values are estimated by daemons. In reality, the actual time delta might be completely different. How can we validate these numbers?</span></p>
<h2><span style="font-weight: 400;">Pulse per second (1PPS)</span></h2>
<p><span style="font-weight: 400;">We can extract an analog signal from the atomic clock (actually the internal timing circuitry from a Stratum 1 device). This signal is called 1PPS, which stands for 1 pulse per second; it generates a pulse over the coaxial cable at the beginning of every second. It is a popular and accurate method of synchronization. We can generate the same pulse from an NTP server and then compare to see how off the phases are. The challenge here is that not all servers support 1PPS, so this requires specialized network extension cards.</span></p>
<p><span style="font-weight: 400;">Our first attempt was pretty much manual. It involved an oscilloscope to display a phase shift. </span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14164 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Oscilloscope-with-PPS-signal.jpg?w=916" alt="Our first attempt was pretty much manual. It involved an oscilloscope to display a phase shift. " width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Oscilloscope-with-PPS-signal.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Oscilloscope-with-PPS-signal.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Oscilloscope-with-PPS-signal.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Oscilloscope-with-PPS-signal.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Oscilloscope-with-PPS-signal.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Oscilloscope-with-PPS-signal.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Oscilloscope-with-PPS-signal.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Oscilloscope-with-PPS-signal.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">Within 10 minutes of measurements we estimated the offset to be approximately 3.5 ms for ntpd, sometimes jumping to 10 ms.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14167 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Grid.jpg?w=916" alt="Within 10 minutes of measurements we estimated the offset to be approximately 3.5 ms for ntpd, sometimes jumping to 10 ms." width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Grid.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Grid.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Grid.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Grid.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Grid.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Grid.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Grid.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Grid.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">This test would be extremely hard and impractical to implement at Facebook scale. A better test involves connecting 1PPS output of the test server back into 1PPS input of the Stratum 1 device itself and monitoring the difference.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14169 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-without-a-PPS-signal.jpg?w=916" alt="A better test involves connecting 1PPS output of the test server back into 1PPS input of the Stratum 1 device itself and monitoring the difference." width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-without-a-PPS-signal.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-without-a-PPS-signal.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-without-a-PPS-signal.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-without-a-PPS-signal.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-without-a-PPS-signal.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-without-a-PPS-signal.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-without-a-PPS-signal.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-without-a-PPS-signal.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">This method has all the upsides of the previous 1PPS measurements but without the downsides of operating an oscilloscope in a data center. As this test runs continuously, we are able to take measurements and verify the real NTP offset at any point in time.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14199 size-full" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-04-1.jpg" alt="As this test runs continuously, we are able to take measurements and verify the real NTP offset at any point in time." width="2000" height="1125" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-04-1.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-04-1.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-04-1.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-04-1.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-04-1.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-04-1.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-04-1.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-04-1.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">These two measurements give us a very good understanding of a real NTP offset with a very small estimated error by 1PPS in just nanoseconds, mostly added by the cable length.</span></p>
<p><span style="font-weight: 400;">The disadvantages of these methods are:</span></p>
<ul>
<li style="font-weight: 400;"><span style="font-weight: 400;">Cabling: Coaxial cable is required for such measurements. Doing spot checks in different data centers requires changes in the data center design, which is challenging.</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">Custom hardware: Not all network cards have 1PPS outputs. Tests like these require unique network cards and servers.</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">Stratum 1 device with 1PPS input.</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">1PPS software on the server: To run the test, we had to install ntpd on our test server. This daemon adds extra error as it works in user space and is scheduled by Linux.</span></li>
</ul>
<h2><span style="font-weight: 400;">Dedicated testing device</span></h2>
<p><span style="font-weight: 400;">There are devices on the market that can perform accuracy checks. They contain GNSS receivers, have an atomic clock, have multiple 1PPS and network interfaces, and also act as NTP clients. This allows us to perform the same checks using NTP protocol directly. The received NTP packet is recorded with a very accurate timestamp from the atomic clock/GNSS receiver.</span></p>
<p><span style="font-weight: 400;">Here is the typical setup:</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14170 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Yellow-Stratum-Satellite-1.jpg?w=916" alt="Here is the typical NTP testing setup:" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Yellow-Stratum-Satellite-1.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Yellow-Stratum-Satellite-1.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Yellow-Stratum-Satellite-1.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Yellow-Stratum-Satellite-1.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Yellow-Stratum-Satellite-1.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Yellow-Stratum-Satellite-1.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Yellow-Stratum-Satellite-1.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Yellow-Stratum-Satellite-1.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;"><img loading="lazy" decoding="async" class="alignnone size-full wp-image-14188" src="https://engineering.fb.com/wp-content/uploads/2020/03/Lab.jpg" alt="Our initial ad-hoc NTP test setup is shown. " width="2000" height="1125" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Lab.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Lab.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Lab.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Lab.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Lab.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Lab.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Lab.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Lab.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /><br />
Our initial ad-hoc test setup is shown above. There are several advantages when measuring with this device:</span></p>
<ul>
<li style="font-weight: 400;"><span style="font-weight: 400;">It does not require extra 1PPS cabling. We still need to discipline the atomic clock, but this can be done using GNSS or the Stratum 1 device itself and a short cable when in the same rack.</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">It uses the atomic clock’s data to stamp transmitted and received network packets. This makes the operating system’s influence negligible, with the error rate in nanoseconds.</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">It supports both NTP and PTP (Precision Time Protocol).</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">The device is portable — we can move it between locations to perform spot checks.</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">The device uses its own format for data points, but it can export data to CSV, which enables us to export data to our internal data standard.</span></li>
</ul>
<h3><span style="font-weight: 400;">NTPD measurements</span><span style="font-weight: 400;"><br />
</span></h3>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14200 size-full" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-05.jpg" alt="This is similar to what we see with daemon estimates and 1PPS measurements. At first, we see a 10 ms drop, which is slowly corrected to +/-1 ms." width="2000" height="1125" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-05.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-05.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-05.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-05.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-05.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-05.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-05.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-05.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">This looks pretty similar to what we’ve seen with daemon estimates and 1PPS measurements. At first, we see a 10 ms drop, which is slowly corrected to +/-1 ms. Interestingly enough, this 10 ms drop is pretty persistent and repeats after every restart.</span></p>
<h3><span style="font-weight: 400;">Chrony measurements</span></h3>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14202 size-full" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-06.jpg" alt="This is 10 to 100 times better than ntpd." width="2000" height="1125" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-06.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-06.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-06.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-06.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-06.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-06.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-06.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-06.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">This looks pretty similar to the chrony daemon estimates. It’s 10 to 100 times better than ntpd.</span></p>
<h3><span style="font-weight: 400;">Chrony with hardware timestamps</span></h3>
<p><span style="font-weight: 400;">Chrony has greatly improved the offset, which can be seen from the estimated 1PPS and lab device values. But there is more: Chrony supports hardware timestamps. According to </span><a href="https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/system_administrators_guide/sect-hw_timestamping"><span style="font-weight: 400;">the documentation</span></a><span style="font-weight: 400;">, it claims to bring accuracy to within hundreds of nanoseconds.</span></p>
<p><span style="font-weight: 400;">Let’s look at the NTP packet structure in the NTP client-server communication over network. The initial client’s NTP packet contains the transmit timestamp field.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14173 size-medium" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Unfilled-Server.jpg?w=916" alt="the NTP packet structure in the NTP client-server communication over network" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Unfilled-Server.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Unfilled-Server.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Unfilled-Server.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Unfilled-Server.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Unfilled-Server.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Unfilled-Server.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Unfilled-Server.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Unfilled-Server.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">The server fills the rest of the fields (e.g., receive timestamp), preserves the client’s transmit timestamp as originate timestamp, and sends it back to the client.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone size-medium wp-image-14174" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Filled-Server.jpg?w=916" alt="" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Filled-Server.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Filled-Server.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Filled-Server.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Filled-Server.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Filled-Server.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Filled-Server.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Filled-Server.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Filled-Server.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">This behavior can be verified using tcpdump:</span></p>
<pre class="line-numbers"><code class="language-none">08:37:31.489921 IP6 (hlim 127, next-header UDP (17) payload length: 56) client.36915 &gt; time1.facebook.com.ntp: [bad udp cksum 0xf5d2 -&gt; 0x595e!] NTPv4, length 48
Client, Leap indicator: clock unsynchronized (192), Stratum 0 (unspecified), poll 3 (8s), precision -6
Root Delay: 1.000000, Root dispersion: 1.000000, Reference-ID: (unspec)
Reference Timestamp: 0.000000000
Originator Timestamp: 0.000000000
Receive Timestamp: 0.000000000
Transmit Timestamp: 3783170251.489887309 (2019/11/19 08:37:31)
Originator - Receive Timestamp: 0.000000000
Originator - Transmit Timestamp: 3783170251.489887309 (2019/11/19 08:37:31)
...
08:37:31.490923 IP6 (hlim 52, next-header UDP (17) payload length: 56) time1.facebook.com.ntp &gt; server.36915: [udp sum ok] NTPv4, length 48
Server, Leap indicator: (0), Stratum 1 (primary reference), poll 3 (8s), precision -32
Root Delay: 0.000000, Root dispersion: 0.000152, Reference-ID: FB
Reference Timestamp: 3783169800.000000000 (2019/11/19 08:30:00)
Originator Timestamp: 3783170251.489887309 (2019/11/19 08:37:31)
Receive Timestamp: 3783170251.490613035 (2019/11/19 08:37:31)
Transmit Timestamp: 3783170251.490631213 (2019/11/19 08:37:31)
Originator - Receive Timestamp: +0.000725725
Originator - Transmit Timestamp: +0.000743903
...</code></pre>
<p><span style="font-weight: 400;">The client will get the packet, append yet another receive timestamp, and calculate the offset using the following formula from the NTP RFC #958:</span></p>
<p><code class="language-none">c = (t2 - t1 + t3 - t4)/2<br />
</code><br />
However, Linux is not a real-time operating system, and it has more than one process to run. So, when the transmit timestamp is filled and <code>Write()</code> is called, there is no guarantee that data is sent over the network immediately:<br />
<img loading="lazy" decoding="async" class="alignnone size-medium wp-image-14175" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Network-Card-Filled.jpg?w=916" alt="" width="916" height="515" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Network-Card-Filled.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Network-Card-Filled.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Network-Card-Filled.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Network-Card-Filled.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Network-Card-Filled.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Network-Card-Filled.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Network-Card-Filled.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time-Infra-Network-Card-Filled.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">Additionally, if the machine does not have much traffic, an ARP request might be required before sending the NTP packet. This all stays unaccountable and gets into estimated error. Luckily, chrony supports hardware timestamps. With these, chrony on the other end can determine with high precision when the packet was processed by the network interface. There is still a delay between the moment the network card stamps the packet and when it actually leaves it, but it’s less than 10 ns. </span></p>
<p>Remember this <code>chronyc sources</code> output from before?</p>
<pre class="line-numbers"><code class="language-none">^+ server4 2 6 377 60 +11ns[-3913ns] +/- 193us</code></pre>
<p><span style="font-weight: 400;">Chrony reports an offset of 11 ns. This is the result of hardware timestamps being enabled on involved parties. However, the estimated error is in the range of hundreds of microseconds. Not every network card supports hardware timestamps, but they are becoming more popular, so chances are the network card already supports them.</span></p>
<p>To verify support for hardware timestamps, simply run the ethtool check so you can see hardware-transmit and hardware-receive capabilities.</p>
<pre class="line-numbers"><code class="language-none">ethtool -T eth0
Time stamping parameters for eth0:
Capabilities:
hardware-transmit (SOF_TIMESTAMPING_TX_HARDWARE)
hardware-receive (SOF_TIMESTAMPING_RX_HARDWARE)
hardware-raw-clock (SOF_TIMESTAMPING_RAW_HARDWARE)</code></pre>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14204 size-full" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-07-1.jpg" alt="By using the lab device to test the same hardware, we were able to achieve different results, where ntpd showed between -10 ms and 3 ms (13 ms difference), chrony showed between -200 µs and 200 µs" width="2000" height="1125" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-07-1.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-07-1.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-07-1.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-07-1.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-07-1.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-07-1.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-07-1.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-07-1.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">By using the lab device to test the same hardware, we were able to achieve different results, where ntpd showed between -10 ms and 3 ms (13 ms difference), chrony showed between -200 µs and 200 µs, and in most cases, chrony with enabled hardware timestamps showed between -100 µs and 100 µs. This confirms that daemon estimates might be inaccurate.</span></p>
<h3><span style="font-weight: 400;">Public services</span></h3>
<p><span style="font-weight: 400;">All these measurements were done in our internal controlled data center networks. Let’s see what happens when we run this device against our public NTP service and some other well-known public NTP service providers: </span></p>
<p><img loading="lazy" decoding="async" class="alignnone wp-image-14205 size-full" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-08-1.jpg" alt="Facebook public NTP service compared to other well-known public NTP service providers " width="2000" height="1125" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-08-1.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-08-1.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-08-1.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-08-1.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-08-1.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-08-1.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-08-1.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-08-1.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">The results of these measurements depend heavily on the network path as well as the speed and quality of the network connection. These measurements were performed multiple times against services from different locations and across a variety of Wi-Fi and LAN networks not associated with Facebook. We can see that not only is our public NTP endpoint competitive with other popular providers, but in some cases, it also outperforms them.</span></p>
<h2><span style="font-weight: 400;">Public NTP design decisions</span></h2>
<p><span style="font-weight: 400;">Once we succeeded in bringing internal precision to the sub-millisecond level, we launched a public NTP service, which can be used by setting time.facebook.com as the NTP server. We run this public NTP service out of our network PoPs (points of presence). To preserve privacy, we do not fingerprint devices by IP address. Having five independent geographically distributed endpoints helps us provide better service — even in the event of a network path failure. So we provide five endpoints: </span></p>
<ul>
<li style="font-weight: 400;"><span style="font-weight: 400;">time1.facebook.com</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">time2.facebook.com</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">time3.facebook.com</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">time4.facebook.com</span></li>
<li style="font-weight: 400;"><span style="font-weight: 400;">time5.facebook.com</span></li>
</ul>
<p><span style="font-weight: 400;">Each of these endpoints terminates in a different geographic location, which has a positive effect on both reliability and time precision. </span></p>
<p><span style="font-weight: 400;">time2.facebook.com network path:</span></p>
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-14192" src="https://engineering.fb.com/wp-content/uploads/2020/03/code1.jpg" alt="time2.facebook.com network path:" width="2000" height="642" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/code1.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/code1.jpg?resize=916,294 916w, https://engineering.fb.com/wp-content/uploads/2020/03/code1.jpg?resize=768,247 768w, https://engineering.fb.com/wp-content/uploads/2020/03/code1.jpg?resize=1024,329 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/code1.jpg?resize=1536,493 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/code1.jpg?resize=96,31 96w, https://engineering.fb.com/wp-content/uploads/2020/03/code1.jpg?resize=192,62 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<p><span style="font-weight: 400;">time3.facebook.com network path:</span></p>
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-14193" src="https://engineering.fb.com/wp-content/uploads/2020/03/code2.jpg" alt="time3.facebook.com network path:" width="1554" height="416" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/code2.jpg 1554w, https://engineering.fb.com/wp-content/uploads/2020/03/code2.jpg?resize=916,245 916w, https://engineering.fb.com/wp-content/uploads/2020/03/code2.jpg?resize=768,206 768w, https://engineering.fb.com/wp-content/uploads/2020/03/code2.jpg?resize=1024,274 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/code2.jpg?resize=1536,411 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/code2.jpg?resize=96,26 96w, https://engineering.fb.com/wp-content/uploads/2020/03/code2.jpg?resize=192,51 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<h3><span style="font-weight: 400;">Leap-second smearing</span></h3>
<p><span style="font-weight: 400;">Ntpd uses a </span><a href="https://data.iana.org/time-zones/tzdb-2018a/leap-seconds.list"><span style="font-weight: 400;">leap-seconds.list</span></a><span style="font-weight: 400;"> file that’s published in advance. Using this file, smearing can start early and the time will be correct by the time the leap second actually occurs.</span></p>
<p><span style="font-weight: 400;">Chrony relies on the leap-second indicator published by GNSS hours in advance. When the leap second event actually happens at 00:00 UTC, it starts to smear throughout a specified time period. With Facebook public NTP, we decided to go with the chrony approach and smear the leap second after the event over approximately 18 hours.</span></p>
<p><span style="font-weight: 400;">As smearing happens on many Stratum 2 servers at the same time, it’s important for the steps to be as similar as possible. Smoother sine-curve smearing is also safer for the applications.</span></p>
<p><img loading="lazy" decoding="async" class="alignnone size-full wp-image-14194" src="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-10.jpg" alt="With Facebook public NTP, we decided to go with the chrony approach and smear the leap second after the event over approximately 18 hours." width="2000" height="1125" srcset="https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-10.jpg 2000w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-10.jpg?resize=580,326 580w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-10.jpg?resize=916,515 916w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-10.jpg?resize=768,432 768w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-10.jpg?resize=1024,576 1024w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-10.jpg?resize=1536,864 1536w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-10.jpg?resize=96,54 96w, https://engineering.fb.com/wp-content/uploads/2020/03/Time_Infra_03-10.jpg?resize=192,108 192w" sizes="auto, (max-width: 992px) 100vw, 62vw" /></p>
<h2><span style="font-weight: 400;">What we’ve learned</span></h2>
<p><span style="font-weight: 400;">Measuring time is challenging. Both ntpd and chrony provide estimates that are somewhat true. In general, if you want to monitor the real offset, we recommend using 1PPS or external devices with GNSS receivers and an atomic clock. </span></p>
<p><span style="font-weight: 400;">In comparing ntpd with chrony, our measurements indicate that chrony is far more precise, which is why we’ve migrated our infra to chrony and launched a public NTP service. We’ve found that the effort to migrate is worth the immediate improvement in precision from tens of milliseconds to hundreds of microseconds. </span></p>
<p><span style="font-weight: 400;">Using hardware timestamps can further improve precision by two orders of magnitude. Despite its improvements, NTP has its own limitations, so evaluating PTP can take your precision to the next level.</span></p>
<p><em>We&#8217;d like to thank Andrei Lukovenko, Alexander Bulimov, Rok Papez, and Yaroslav Kolomiiets for their support on this project.</em></p>
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-facebook"><a rel="nofollow noopener noreferrer"
data-shared="sharing-facebook-14157"
class="share-facebook sd-button share-icon"
href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/?share=facebook"
target="_blank"
aria-labelledby="sharing-facebook-14157"
>
<span id="sharing-facebook-14157" hidden>Share on Facebook (Opens in new window)</span>
<span>Facebook</span>
</a></li><li class="share-threads"><a rel="nofollow noopener noreferrer"
data-shared="sharing-threads-14157"
class="share-threads sd-button share-icon"
href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/?share=threads"
target="_blank"
aria-labelledby="sharing-threads-14157"
>
<span id="sharing-threads-14157" hidden>Share on Threads (Opens in new window)</span>
<span>Threads</span>
</a></li><li class="share-jetpack-whatsapp"><a rel="nofollow noopener noreferrer"
data-shared="sharing-whatsapp-14157"
class="share-jetpack-whatsapp sd-button share-icon"
href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/?share=jetpack-whatsapp"
target="_blank"
aria-labelledby="sharing-whatsapp-14157"
>
<span id="sharing-whatsapp-14157" hidden>Share on WhatsApp (Opens in new window)</span>
<span>WhatsApp</span>
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
data-shared="sharing-linkedin-14157"
class="share-linkedin sd-button share-icon"
href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/?share=linkedin"
target="_blank"
aria-labelledby="sharing-linkedin-14157"
>
<span id="sharing-linkedin-14157" hidden>Share on LinkedIn (Opens in new window)</span>
<span>LinkedIn</span>
</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
data-shared="sharing-reddit-14157"
class="share-reddit sd-button share-icon"
href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/?share=reddit"
target="_blank"
aria-labelledby="sharing-reddit-14157"
>
<span id="sharing-reddit-14157" hidden>Share on Reddit (Opens in new window)</span>
<span>Reddit</span>
</a></li><li class="share-x"><a rel="nofollow noopener noreferrer"
data-shared="sharing-x-14157"
class="share-x sd-button share-icon"
href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/?share=x"
target="_blank"
aria-labelledby="sharing-x-14157"
>
<span id="sharing-x-14157" hidden>Share on X (Opens in new window)</span>
<span>X</span>
</a></li><li class="share-bluesky"><a rel="nofollow noopener noreferrer"
data-shared="sharing-bluesky-14157"
class="share-bluesky sd-button share-icon"
href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/?share=bluesky"
target="_blank"
aria-labelledby="sharing-bluesky-14157"
>
<span id="sharing-bluesky-14157" hidden>Share on Bluesky (Opens in new window)</span>
<span>Bluesky</span>
</a></li><li class="share-mastodon"><a rel="nofollow noopener noreferrer"
data-shared="sharing-mastodon-14157"
class="share-mastodon sd-button share-icon"
href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/?share=mastodon"
target="_blank"
aria-labelledby="sharing-mastodon-14157"
>
<span id="sharing-mastodon-14157" hidden>Share on Mastodon (Opens in new window)</span>
<span>Mastodon</span>
</a></li><li class="share-custom share-custom-hackernews"><a rel="nofollow noopener noreferrer"
data-shared="sharing-custom-14157"
class="share-custom share-custom-hackernews sd-button share-icon"
href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/?share=custom-1699562127"
target="_blank"
aria-labelledby="sharing-custom-14157"
>
<span id="sharing-custom-14157" hidden>Share on Hacker News (Opens in new window)</span>
<span style="background-image:url(&quot;https://news.ycombinator.com/y18.svg&quot;);">Hacker News</span>
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
data-shared="sharing-email-14157"
class="share-email sd-button share-icon"
href="mailto:?subject=%5BShared%20Post%5D%20Building%20a%20more%20accurate%20time%20service%20at%20Facebook%20scale&#038;body=https%3A%2F%2Fengineering.fb.com%2F2020%2F03%2F18%2Fproduction-engineering%2Fntp-service%2F&#038;share=email"
target="_blank"
aria-labelledby="sharing-email-14157"
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you&#039;re having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="ee26f6398b" data-email-share-track-url="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/?share=email">
<span id="sharing-email-14157" hidden>Email a link to a friend (Opens in new window)</span>
<span>Email</span>
</a></li><li class="share-end"></li></ul></div></div></div>
</div><!-- .entry-content -->
</article><!-- #post-## -->
<footer class="entry-footer">
</footer><!-- .entry-footer -->
<div class="fb-like"
data-href="https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/"
data-width=""
data-layout="standard"
data-action="like"
data-size="small"
data-share="true">
</div>
<div class="related-articles">
<div class="related-articles__header">
<h3 class="related-articles__header__headline">Read More in Production Engineering</h3>
<a
aria-label="View all articles in this Production Engineering"
class="related-articles__header__link"
href="https://engineering.fb.com/category/production-engineering/"
>
View All </a>
</div>
<div class="article-grids related-articles__grid">
<div class="row">
<div class="article-grid col-12 col-md-4">
<article class="post post-14157 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24394">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/08/Meta-MTIA-300-Hero-final.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" title="Meta MTIA 300 - Hero - final" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-08-24T10:45:52-07:00">
AUG 24, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/08/24/networking-traffic/mtia-300-meta-training-chip-built-in-nics/" rel="bookmark" aria-label="Read More about MTIA 300: Meta&#8217;s First Training Chip with Built-in NICs and Communication-Offloading Engines">
MTIA 300: Meta&#8217;s First Training Chip with Built-in NICs and Communication-Offloading Engines </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-14157 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-23996">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/05/Meta-Mult-stage-Ads-Ranking.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-08-05T12:20:20-07:00">
AUG 5, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/08/05/ml-applications/from-user-sequences-to-scaling-laws-a-multi-stage-architecture-for-metas-ads-ranking/" rel="bookmark" aria-label="Read More about From User Sequences to Scaling Laws: A Multi-Stage Architecture for Meta’s Ads Ranking">
From User Sequences to Scaling Laws: A Multi-Stage Architecture for Meta’s Ads Ranking </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-14157 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24150">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Metas-AI-Storage-Blueprint-at-Scale-Hero-1.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-07-01T09:00:36-07:00">
JUL 1, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/07/01/data-infrastructure/metas-ai-storage-blueprint-at-scale/" rel="bookmark" aria-label="Read More about Meta&#8217;s AI Storage Blueprint at Scale">
Meta&#8217;s AI Storage Blueprint at Scale </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-14157 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24128">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/10-Years-Meta-Python-HERO-large.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-30T09:00:46-07:00">
JUN 30, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/30/open-source/10-years-of-metas-commitment-to-python/" rel="bookmark" aria-label="Read More about 10 Years of Meta’s Commitment to Python">
10 Years of Meta’s Commitment to Python </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-14157 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24113">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Meta-Tech-Podcast-How-Meta-Built-Ultra-Narrow-Batteries-for-AI-Glasses-HERO.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-23T09:00:38-07:00">
JUN 23, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/23/production-engineering/how-meta-built-ultra-narrow-batteries-for-ai-glasses-meta-tech-podcast/" rel="bookmark" aria-label="Read More about How Meta Engineered Ultra-Narrow Batteries for AI Glasses">
How Meta Engineered Ultra-Narrow Batteries for AI Glasses </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
<div class="article-grid col-12 col-md-4">
<article class="post post-14157 type-post status-publish format-standard has-post-thumbnail hentry category-production-engineering fb_content_type-article" id="post-24022">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2026/06/Validating-PowerLoss-Readiness-Meta-hero.png?w=580&amp;h=326&amp;crop=1" class="attachment-thumbnail size-thumbnail" alt="" loading="lazy" />
<header class="entry-header">
<div class="entry-header__top">
<span class="posted-on">
<time class="entry-date published" datetime="2026-06-03T10:00:44-07:00">
JUN 3, 2026 </time>
</span>
</div>
<div class="entry-title">
<div>
<a href="https://engineering.fb.com/2026/06/03/data-center-engineering/lights-out-systems-on-validating-instant-power-loss-readiness/" rel="bookmark" aria-label="Read More about Lights Out, Systems On: Validating Instant Power Loss Readiness">
Lights Out, Systems On: Validating Instant Power Loss Readiness </a>
</div>
</div>
</header><!-- .entry-header -->
</article><!-- #post-## -->
</div><!-- /.article-grid -->
</div><!-- /.row -->
</div><!--/.article-grids -->
</div><!-- /.related-articles -->
<!-- include the carousel nav here. -->
</main><!-- #main -->
</div><!-- #primary -->
<!-- Do the right sidebar check. -->
<div class="col-md-4 widget-area" id="right-sidebar" role="complementary">
<aside id="fb_related_post_widget-4" class="widget widget_fb_related_post_widget"><h3 class="widget-title">Related Posts</h3><hr>
<div class="aside-related-projects"><div class="article-grid-redesign">
<article class="post-11758 post type-post status-publish has-post-thumbnail hentry" id="post-11758">
<a href="https://engineering.fb.com/2018/12/21/security/dns-over-tls/" rel="bookmark" aria-label="Read More about DNS over TLS: Encrypting DNS end-to-end">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2018/12/DoT-Hero.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2018/12/DoT-Hero.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2018/12/DoT-Hero.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Dec 21, 2018">Dec 21, 2018</time></span>
<h4 class="entry-title">DNS over TLS: Encrypting DNS end-to-end</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div><div class="article-grid-redesign">
<article class="post-7441 post type-post status-publish has-post-thumbnail hentry" id="post-7441">
<a href="https://engineering.fb.com/2018/02/12/production-engineering/how-production-engineers-support-global-events-on-facebook/" rel="bookmark" aria-label="Read More about How production engineers support global events on Facebook">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2018/02/fb-live-new-years.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2018/02/fb-live-new-years.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2018/02/fb-live-new-years.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="Feb 12, 2018">Feb 12, 2018</time></span>
<h4 class="entry-title">How production engineers support global events on Facebook</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div><div class="article-grid-redesign">
<article class="post-13060 post type-post status-publish has-post-thumbnail hentry" id="post-13060">
<a href="https://engineering.fb.com/2019/05/28/data-infrastructure/dhcplb-server/" rel="bookmark" aria-label="Read More about Extending DHCPLB: The path from load balancer to server">
<img width="580" height="326" src="https://engineering.fb.com/wp-content/uploads/2019/05/DHCP.jpg?w=580&#038;h=326&#038;crop=1" class="wp-post-image" alt="" srcset="https://engineering.fb.com/wp-content/uploads/2019/05/DHCP.jpg?w=580&#038;h=326&#038;crop=1 580w, https://engineering.fb.com/wp-content/uploads/2019/05/DHCP.jpg?w=580&#038;h=326&#038;crop=1 326w" sizes="(max-width: 580px) 100vw, 580px" loading="lazy">
<header class="entry-header">
<span class="posted-on"><time class="entry-date published" datetime="May 28, 2019">May 28, 2019</time></span>
<h4 class="entry-title">Extending DHCPLB: The path from load balancer to server</h4>
</header><!-- .entry-header -->
</a>
</article><!-- #post-## -->
</div></div></aside><aside id="wpb_widget-3" class="widget widget_wpb_widget"><h3 class="widget-title">Related Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2176819739360965/>
<div class="job-description">
Production Engineering
</div>
<div class="job-location">
BELLEVUE, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2176819739360965/>
<div class="job-description">
Production Engineering
</div>
<div class="job-location">
MENLO PARK, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2176819739360965/>
<div class="job-description">
Production Engineering
</div>
<div class="job-location">
BOSTON, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2176819739360965/>
<div class="job-description">
Production Engineering
</div>
<div class="job-location">
NEW YORK, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/1990732088273422/>
<div class="job-description">
Network Production Engineer, DC Frontier
</div>
<div class="job-location">
MENLO PARK, US
</div>
</a>
</li> </ul>
<a href="https://www.metacareers.com">
<div class="btn available-positions-btn">
See All Jobs </div>
</a>
</aside>
</div><!-- #secondary -->
<!-- <php get_template_part('content-single-side-bar'); ?> -->
<!-- Do the right sidebar check. -->
</div><!-- Container end -->
</div><!-- Wrapper end -->
</div>
<!-- ******************* The Footer Full-width Widget Area ******************* -->
<div class="wrapper wrapper-community-outer">
<div class="wrapper-community">
<div class="community" tabindex="-1">
<div class="row">
<!-- community section is generated through WordPress widget -->
<div id="wpb_widget-2" class="footer-widget widget_wpb_widget widget-count-3 col-md-4"><h3 class="widget-title">Available Positions</h3><hr><ul class="available-positions-list"><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2176819739360965/>
<div class="job-description">
Production Engineering
</div>
<div class="job-location">
BELLEVUE, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2176819739360965/>
<div class="job-description">
Production Engineering
</div>
<div class="job-location">
MENLO PARK, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2176819739360965/>
<div class="job-description">
Production Engineering
</div>
<div class="job-location">
BOSTON, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/2176819739360965/>
<div class="job-description">
Production Engineering
</div>
<div class="job-location">
NEW YORK, US
</div>
</a>
</li><li class="available-positions-item">
<a href=https://www.metacareers.com/jobs/1990732088273422/>
<div class="job-description">
Network Production Engineer, DC Frontier
</div>
<div class="job-location">
MENLO PARK, US
</div>
</a>
</li> </ul>
<a href="https://www.metacareers.com">
<div class="btn available-positions-btn">
See All Jobs </div>
</a>
</div><!-- .footer-widget --><div id="custom_html-2" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Technology at Meta</h3><div class="textwidget custom-html-widget"><ul class="stay-connected-lists">
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-fb-engineering" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Engineering at Meta - X
</div>
<a onclick='return twitterFollow("Meta_Engineers")'>
<button class="btn stay-connected-btn">
Follow
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-AI" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
AI at Meta
</div>
<a href="https://ai.meta.com/blog/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta Quest Blog
</div>
<a href="https://www.meta.com/blog/quest/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-developers" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta for Developers
</div>
<a href="https://developers.facebook.com/">
<button class="btn stay-connected-btn">
Read
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="footer-fb-engineering stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/meta_logo.png" alt="footer-bug-bounty" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
Meta Bug Bounty
</div>
<a href="https://bugbounty.meta.com/">
<button class="btn stay-connected-btn">
Learn more
</button>
</a>
</div>
</li>
<li class="stay-connected-item">
<img class="stay-connected-item-left" src="/wp-content/themes/code-fb-com/img/rss.png" alt="footer-rss" loading="lazy" width="70" height="70">
<div class="stay-connected-item-right">
<div class="stay-connected-label">
RSS
</div>
<a href="https://code.facebook.com/posts/rss/">
<button class="btn stay-connected-btn">
Subscribe
</button>
</a>
</div>
</li>
</ul>
</div></div><!-- .footer-widget --><div id="custom_html-3" class="widget_text footer-widget widget_custom_html widget-count-3 col-md-4"><h3 class="widget-title">Open Source</h3><div class="textwidget custom-html-widget"><p class="opensource-desc">
Meta believes in building community through open source technology. Explore our latest projects in Artificial Intelligence, Data Infrastructure, Development Tools, Front End, Languages, Platforms, Security, Virtual Reality, and more.
</p>
<ul class="opensource-cats">
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/android.png" alt="android" loading="lazy" width="50" height="59">
<div class="opensource-cap">
ANDROID
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/ios.png" alt="ios" loading="lazy" width="48" height="58">
<div class="opensource-cap">
iOS
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/web.png" alt="web" loading="lazy" width="55" height="49">
<div class="opensource-cap">
WEB
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/backend.png" alt="backend" loading="lazy" width="56" height="51">
<div class="opensource-cap">
BACKEND
</div>
</li>
<li class="opensource-cat">
<img src="/wp-content/themes/code-fb-com/img/hardware.png" alt="hardware" loading="lazy" width="55" height="44">
<div class="opensource-cap">
HARDWARE
</div>
</li>
</ul>
<button class="btn opensource-btn" onclick="window.location.href='https://opensource.fb.com/'">
Learn More
</button>
</div></div><!-- .footer-widget -->
</div>
</div>
</div>
</div><!-- #wrapper-footer-full -->
<div class="wrapper-footer" id="wrapper-footer">
<div class="inner-container">
<div class="footer">
<div class="row">
<div class="col-md-12">
<footer class="meta-footer" id="meta-footer">
<div class="site-info row justify-content-between">
<div class="col-md-5 meta-logo-container">
<a href="https://about.facebook.com/">
<img src="https://engineering.fb.com/wp-content/themes/code-fb-com/img/meta_logo_full.svg" width="89" height="18" alt="Meta" />
</a>
<div id="text-2" class="widget widget_text"> <div class="textwidget"><p>Engineering at Meta is a technical news resource for engineers interested in how we solve large-scale technical challenges at Meta.</p>
</div>
</div> </div>
<div class="col-md-6 footer-meta-links-container">
<ul class="meta-footer-links">
<li class="footer-link"><a href="https://engineering.fb.com">Home</a></li><li class="footer-link"><a href="https://about.meta.com/">Company Info</a></li><li class="footer-link"><a href="https://www.metacareers.com/?ref=engineering.fb.com">Careers</a></li> </ul>
</div>
</div>
</footer>
</div>
</div>
</div>
</div>
<div class="footer">
<div class="row">
<div class="col-md-12">
<footer class="site-footer" id="colophon">
<div class="inner-container">
<div class="site-info row justify-content-between">
<div class="col-md-4 copyright-container">
<div class="copyright"> © 2026 Meta</div>
</div>
<!-- static footer -->
<div class="col-md-7 footer-links-container">
<ul class="footer-links">
<li class="footer-link"><a href="https://www.facebook.com/policies">Terms</a></li><li class="footer-link"><a href="https://www.facebook.com/privacy/policy">Privacy</a></li><li class="footer-link"><a href="/privacy">Cookies</a></li><li class="footer-link"><a href="https://www.facebook.com/help">Help</a></li> </ul>
</div>
</div>
</div><!-- .site-info -->
</footer><!-- #colophon -->
</div><!--col end -->
</div><!-- row end -->
</div><!-- //.inner-container -->
</div><!-- wrapper end -->
</div><!-- #page we need this extra closing tag here -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/code-fb-com/*","/wp-content/themes/understrap/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<div class="gdprconsent-container" id="GDPRConsentBar">
<div class="gdprconsent-wrapper">
<div class="gdprconsent-content">
<p>To help personalize content, tailor and measure ads and provide a safer experience, we use cookies. By clicking or navigating the site, you agree to allow our collection of information on and off Facebook through cookies. Learn more, including about available controls: <a href="/privacy">Cookie Policy</a></p>
<div class="gdprconsent-button">
<button type="button" data-gdpr-accept>Accept</button>
</div>
</div>
</div>
</div>
<!-- Twitter universal website tag code -->
<script>
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
// Insert Twitter Pixel ID and Standard Event data below
twq('init','o31u0');
twq('track','PageView');
});
}
</script>
<!-- End Twitter universal website tag code -->
<script type="text/javascript">
window.WPCOM_sharing_counts = {"https://engineering.fb.com/2020/03/18/production-engineering/ntp-service/":14157};
</script>
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KZRLGZ8" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=660742068067366&ev=PageView&noscript=1"/></noscript>
<script id="child-understrap-scripts-js-extra">
var loadmore_params = {"restfulURL":"https://engineering.fb.com/wp-json/fb/v1/loadmore","searchText":"","posts":"{\"page\":0,\"year\":2020,\"monthnum\":3,\"day\":18,\"name\":\"ntp-service\",\"category_name\":\"production-engineering\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","post_type":"post"};
var search_params = {"ajaxurl":"https://engineering.fb.com/wp-admin/admin-ajax.php","posts":"{\"page\":0,\"year\":2020,\"monthnum\":3,\"day\":18,\"name\":\"ntp-service\",\"category_name\":\"production-engineering\",\"error\":\"\",\"m\":\"\",\"p\":0,\"post_parent\":\"\",\"subpost\":\"\",\"subpost_id\":\"\",\"attachment\":\"\",\"attachment_id\":0,\"pagename\":\"\",\"page_id\":0,\"second\":\"\",\"minute\":\"\",\"hour\":\"\",\"w\":0,\"tag\":\"\",\"cat\":\"\",\"tag_id\":\"\",\"author\":\"\",\"author_name\":\"\",\"feed\":\"\",\"tb\":\"\",\"paged\":0,\"meta_key\":\"\",\"meta_value\":\"\",\"preview\":\"\",\"s\":\"\",\"sentence\":\"\",\"title\":\"\",\"fields\":\"all\",\"menu_order\":\"\",\"embed\":\"\",\"category__in\":[],\"category__not_in\":[],\"category__and\":[],\"post__in\":[],\"post__not_in\":[],\"post_name__in\":[],\"tag__in\":[],\"tag__not_in\":[],\"tag__and\":[],\"tag_slug__in\":[],\"tag_slug__and\":[],\"post_parent__in\":[],\"post_parent__not_in\":[],\"author__in\":[],\"author__not_in\":[],\"search_columns\":[],\"ignore_sticky_posts\":false,\"suppress_filters\":false,\"cache_results\":true,\"update_post_term_cache\":true,\"update_menu_item_cache\":false,\"lazy_load_term_meta\":true,\"update_post_meta_cache\":true,\"post_type\":\"\",\"posts_per_page\":12,\"nopaging\":false,\"comments_per_page\":\"50\",\"no_found_rows\":false,\"order\":\"DESC\"}","current_page":"1","max_page":"0","article_filter":"2f23ad5ce3"};
var more_articles = {"next_article":{"label":"Next","permalink":"https://engineering.fb.com/2020/04/21/data-infrastructure/turbine/","title":"Turbine: Facebook\u2019s service management platform for stream processing","thumbnail":["https://engineering.fb.com/wp-content/uploads/2020/04/RiB_blue_green_dark.jpg?w=580&h=326&crop=1",580,326,true]},"prev_article":{"label":"Prev","permalink":"https://engineering.fb.com/2020/03/05/developer-tools/incident-tracker/","title":"Preventing performance regressions with Health Compass and Incident Tracker","thumbnail":["https://engineering.fb.com/wp-content/uploads/2020/03/Health_Compass_01.jpg?w=580&h=326&crop=1",580,326,true]}};
//# sourceURL=child-understrap-scripts-js-extra
</script>
<script type="text/javascript" src="https://engineering.fb.com/_static/??/wp-content/themes/code-fb-com/dist/js/child-theme.min.js,/wp-content/themes/code-fb-com/js/prism.js?m=1783017802j" ></script><script id="sharing-js-js-extra">
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":""};
//# sourceURL=sharing-js-js-extra
</script>
<script id="sharing-js-js" src="https://engineering.fb.com/wp-content/mu-plugins/jetpack-16.1/_inc/build/sharedaddy/sharing.min.js?ver=16.1.3"></script>
<script id="sharing-js-js-after">
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-facebook' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-threads' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-threads' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomthreads', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-linkedin' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-x' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-x' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomx', 'menubar=1,resizable=1,width=600,height=350' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-bluesky' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-bluesky' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcombluesky', 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
var windowOpen;
( function () {
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-mastodon' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-mastodon' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), 'wpcommastodon', 'menubar=1,resizable=1,width=460,height=400' );
return false;
}
} );
} )();
//# sourceURL=sharing-js-js-after
</script>
<script id="video-analytics-js-after">
if (window.gdprSafeTrack) {
window.gdprSafeTrack(function() {
let youtube = document.querySelectorAll('[src*="youtube-nocookie"]');
for (let i = 0; i < youtube.length; i++) {
youtube[i].src = youtube[i].src.replace('youtube-nocookie.com', 'youtube.com');
}
let vimeo = document.querySelectorAll('[src*="vimeo"]');
for (let i = 0; i < vimeo.length; i++) {
vimeo[i].src = vimeo[i].src.replace('&dnt=true', '');
}
});
}
//# sourceURL=video-analytics-js-after
</script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://engineering.fb.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://engineering.fb.com/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>