Files
nexus/sreweekly/articles/5/05-holistic-configuration-management-at-facebook-the-morning-paper.html
2026-09-12 17:23:01 +08:00

1018 lines
102 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="profile" href="http://gmpg.org/xfn/11">
<title>Holistic Configuration Management at Facebook &#8211; the morning paper</title>
<meta name='robots' content='max-image-preview:large' />
<!-- Async WordPress.com Remote Login -->
<script id="wpcom_remote_login_js">
var wpcom_remote_login_extra_auth = '';
function wpcom_remote_login_remove_dom_node_id( element_id ) {
var dom_node = document.getElementById( element_id );
if ( dom_node ) { dom_node.parentNode.removeChild( dom_node ); }
}
function wpcom_remote_login_remove_dom_node_classes( class_name ) {
var dom_nodes = document.querySelectorAll( '.' + class_name );
for ( var i = 0; i < dom_nodes.length; i++ ) {
dom_nodes[ i ].parentNode.removeChild( dom_nodes[ i ] );
}
}
function wpcom_remote_login_final_cleanup() {
wpcom_remote_login_remove_dom_node_classes( "wpcom_remote_login_msg" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_validate" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_js" );
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_iframe" );
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_styles" );
}
// Watch for messages back from the remote login
window.addEventListener( "message", function( e ) {
if ( e.origin === "https://r-login.wordpress.com" ) {
var data = {};
try {
data = JSON.parse( e.data );
} catch( e ) {
wpcom_remote_login_final_cleanup();
return;
}
if ( data.msg === 'LOGIN' ) {
// Clean up the login check iframe
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
var id_regex = new RegExp( /^[0-9]+$/ );
var token_regex = new RegExp( /^.*|.*|.*$/ );
if (
token_regex.test( data.token )
&& id_regex.test( data.wpcomid )
) {
// We have everything we need to ask for a login
var script = document.createElement( "script" );
script.setAttribute( "id", "wpcom_remote_login_validate" );
script.src = '/remote-login.php?wpcom_remote_login=validate'
+ '&wpcomid=' + data.wpcomid
+ '&token=' + encodeURIComponent( data.token )
+ '&host=' + window.location.protocol
+ '//' + window.location.hostname
+ '&postid=1235'
+ '&is_singular=1';
document.body.appendChild( script );
}
return;
}
// Safari ITP, not logged in, so redirect
if ( data.msg === 'LOGIN-REDIRECT' ) {
window.location = 'https://wordpress.com/log-in?redirect_to=' + window.location.href;
return;
}
// Safari ITP, storage access failed, remove the request
if ( data.msg === 'LOGIN-REMOVE' ) {
var css_zap = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } @media screen and ( max-width: 782px ) { html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } }';
var style_zap = document.createElement( 'style' );
style_zap.type = 'text/css';
style_zap.appendChild( document.createTextNode( css_zap ) );
document.body.appendChild( style_zap );
var e = document.getElementById( 'wpcom_request_access_iframe' );
e.parentNode.removeChild( e );
document.cookie = 'wordpress_com_login_access=denied; path=/; max-age=31536000';
return;
}
// Safari ITP
if ( data.msg === 'REQUEST_ACCESS' ) {
console.log( 'request access: safari' );
// Check ITP iframe enable/disable knob
if ( wpcom_remote_login_extra_auth !== 'safari_itp_iframe' ) {
return;
}
// If we are in a "private window" there is no ITP.
var private_window = false;
try {
var opendb = window.openDatabase( null, null, null, null );
} catch( e ) {
private_window = true;
}
if ( private_window ) {
console.log( 'private window' );
return;
}
var iframe = document.createElement( 'iframe' );
iframe.id = 'wpcom_request_access_iframe';
iframe.setAttribute( 'scrolling', 'no' );
iframe.setAttribute( 'sandbox', 'allow-storage-access-by-user-activation allow-scripts allow-same-origin allow-top-navigation-by-user-activation' );
iframe.src = 'https://r-login.wordpress.com/remote-login.php?wpcom_remote_login=request_access&origin=' + encodeURIComponent( data.origin ) + '&wpcomid=' + encodeURIComponent( data.wpcomid );
var css = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 46px !important; } * html body { margin-top: 46px !important; } @media screen and ( max-width: 660px ) { html { margin-top: 71px !important; } * html body { margin-top: 71px !important; } #wpcom_request_access_iframe { display: block; height: 71px !important; } } #wpcom_request_access_iframe { border: 0px; height: 46px; position: fixed; top: 0; left: 0; width: 100%; min-width: 100%; z-index: 99999; background: #23282d; } ';
var style = document.createElement( 'style' );
style.type = 'text/css';
style.id = 'wpcom_request_access_styles';
style.appendChild( document.createTextNode( css ) );
document.body.appendChild( style );
document.body.appendChild( iframe );
}
if ( data.msg === 'DONE' ) {
wpcom_remote_login_final_cleanup();
}
}
}, false );
// Inject the remote login iframe after the page has had a chance to load
// more critical resources
window.addEventListener( "DOMContentLoaded", function( e ) {
var iframe = document.createElement( "iframe" );
iframe.style.display = "none";
iframe.setAttribute( "scrolling", "no" );
iframe.setAttribute( "id", "wpcom_remote_login_key" );
iframe.src = "https://r-login.wordpress.com/remote-login.php"
+ "?wpcom_remote_login=key"
+ "&origin=aHR0cHM6Ly9ibG9nLmFjb2x5ZXIub3Jn"
+ "&wpcomid=23592848"
+ "&time=" + Math.floor( Date.now() / 1000 );
document.body.appendChild( iframe );
}, false );
</script>
<link rel='dns-prefetch' href='//s2.wp.com' />
<link rel='dns-prefetch' href='//fonts-api.wp.com' />
<link rel='dns-prefetch' href='//s1.wp.com' />
<link rel='dns-prefetch' href='//s0.wp.com' />
<link rel="alternate" type="application/rss+xml" title="the morning paper &raquo; Feed" href="https://blog.acolyer.org/feed/" />
<link rel="alternate" type="application/rss+xml" title="the morning paper &raquo; Comments Feed" href="https://blog.acolyer.org/comments/feed/" />
<link rel="alternate" type="application/rss+xml" title="the morning paper &raquo; Holistic Configuration Management at&nbsp;Facebook Comments Feed" href="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/feed/" />
<script type="text/javascript">
/* <![CDATA[ */
function addLoadEvent(func) {
var oldonload = window.onload;
if (typeof window.onload != 'function') {
window.onload = func;
} else {
window.onload = function () {
oldonload();
func();
}
}
}
/* ]]> */
</script>
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-2-1' href='/_static/??-eJxtjEsKgDAMBS9kjbqwuBDPUtog9o9J8fpaEQSRBw8GhoEjC50iY2QIRWRf1i0SWOSstHsYQkr1TPFIsKNXjEbkRPyhVhM18J/0m8M3fFPVlzD3UnbTtWG0J2CTNNk=&cssminify=yes' type='text/css' media='all' />
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/common.min.css */
</style>
<style id="wp-block-library-inline-css-extra">
.has-text-align-justify {
text-align:justify;
}
.has-text-align-justify{text-align:justify;}
/*# sourceURL=wp-block-library-inline-css */
</style>
<style id="classic-theme-styles-inline-css">
.wp-block-button__link{background-color:#32373c;border-radius:9999px;box-shadow:none;color:#fff;font-size:1.125em;padding:calc(.667em + 2px) calc(1.333em + 2px);text-decoration:none}.wp-block-file__button{background:#32373c;color:#fff}.wp-block-accordion-heading{margin:0}.wp-block-accordion-heading__toggle{background-color:inherit!important;color:inherit!important}.wp-block-accordion-heading__toggle:not(:focus-visible){outline:none}.wp-block-accordion-heading__toggle:focus,.wp-block-accordion-heading__toggle:hover{background-color:inherit!important;border:none;box-shadow:none;color:inherit;padding:var(--wp--preset--spacing--20,1em) 0;text-decoration:none}.wp-block-accordion-heading__toggle:focus-visible{outline:auto;outline-offset:0}.wp-block-tab:not(.has-text-color){color:inherit!important}.wp-block-tab:not(.has-background){background-color:inherit!important}.wp-block-tab:focus,.wp-block-tab:hover{text-decoration:none}.wp-block-tab:focus-visible{outline:auto;outline-offset:0}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/classic.min.css */
</style>
<style id="coblocks-inline-format-inline-css">
.uppercase{text-transform:uppercase}
/*# sourceURL=coblocks-inline-format-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-12-1' href='/_static/??/wp-content/mu-plugins/core-compat/wp-mediaelement.css,/wp-content/mu-plugins/wpcom-bbpress-premium-themes.css?m=1432920480j&cssminify=yes' type='text/css' media='all' />
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #fff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--color--dark-green: #1c7c7c;--wp--preset--color--dark-gray: #666;--wp--preset--color--medium-gray: #999;--wp--preset--color--light-gray: #eee;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--font-family--albert-sans: 'Albert Sans', sans-serif;--wp--preset--font-family--alegreya: Alegreya, serif;--wp--preset--font-family--arvo: Arvo, serif;--wp--preset--font-family--bodoni-moda: 'Bodoni Moda', serif;--wp--preset--font-family--bricolage-grotesque: 'Bricolage Grotesque', sans-serif;--wp--preset--font-family--cabin: Cabin, sans-serif;--wp--preset--font-family--chivo: Chivo, sans-serif;--wp--preset--font-family--commissioner: Commissioner, sans-serif;--wp--preset--font-family--cormorant: Cormorant, serif;--wp--preset--font-family--courier-prime: 'Courier Prime', monospace;--wp--preset--font-family--crimson-pro: 'Crimson Pro', serif;--wp--preset--font-family--dm-mono: 'DM Mono', monospace;--wp--preset--font-family--dm-sans: 'DM Sans', sans-serif;--wp--preset--font-family--dm-serif-display: 'DM Serif Display', serif;--wp--preset--font-family--domine: Domine, serif;--wp--preset--font-family--eb-garamond: 'EB Garamond', serif;--wp--preset--font-family--epilogue: Epilogue, sans-serif;--wp--preset--font-family--fahkwang: Fahkwang, sans-serif;--wp--preset--font-family--figtree: Figtree, sans-serif;--wp--preset--font-family--fira-sans: 'Fira Sans', sans-serif;--wp--preset--font-family--fjalla-one: 'Fjalla One', sans-serif;--wp--preset--font-family--fraunces: Fraunces, serif;--wp--preset--font-family--gabarito: Gabarito, system-ui;--wp--preset--font-family--ibm-plex-mono: 'IBM Plex Mono', monospace;--wp--preset--font-family--ibm-plex-sans: 'IBM Plex Sans', sans-serif;--wp--preset--font-family--ibarra-real-nova: 'Ibarra Real Nova', serif;--wp--preset--font-family--instrument-serif: 'Instrument Serif', serif;--wp--preset--font-family--inter: Inter, sans-serif;--wp--preset--font-family--josefin-sans: 'Josefin Sans', sans-serif;--wp--preset--font-family--jost: Jost, sans-serif;--wp--preset--font-family--libre-baskerville: 'Libre Baskerville', serif;--wp--preset--font-family--libre-franklin: 'Libre Franklin', sans-serif;--wp--preset--font-family--literata: Literata, serif;--wp--preset--font-family--lora: Lora, serif;--wp--preset--font-family--merriweather: Merriweather, serif;--wp--preset--font-family--montserrat: Montserrat, sans-serif;--wp--preset--font-family--newsreader: Newsreader, serif;--wp--preset--font-family--noto-sans-mono: 'Noto Sans Mono', sans-serif;--wp--preset--font-family--nunito: Nunito, sans-serif;--wp--preset--font-family--open-sans: 'Open Sans', sans-serif;--wp--preset--font-family--overpass: Overpass, sans-serif;--wp--preset--font-family--pt-serif: 'PT Serif', serif;--wp--preset--font-family--petrona: Petrona, serif;--wp--preset--font-family--piazzolla: Piazzolla, serif;--wp--preset--font-family--playfair-display: 'Playfair Display', serif;--wp--preset--font-family--plus-jakarta-sans: 'Plus Jakarta Sans', sans-serif;--wp--preset--font-family--poppins: Poppins, sans-serif;--wp--preset--font-family--raleway: Raleway, sans-serif;--wp--preset--font-family--roboto: Roboto, sans-serif;--wp--preset--font-family--roboto-slab: 'Roboto Slab', serif;--wp--preset--font-family--rubik: Rubik, sans-serif;--wp--preset--font-family--rufina: Rufina, serif;--wp--preset--font-family--sora: Sora, sans-serif;--wp--preset--font-family--source-sans-3: 'Source Sans 3', sans-serif;--wp--preset--font-family--source-serif-4: 'Source Serif 4', serif;--wp--preset--font-family--space-mono: 'Space Mono', monospace;--wp--preset--font-family--syne: Syne, sans-serif;--wp--preset--font-family--texturina: Texturina, serif;--wp--preset--font-family--urbanist: Urbanist, sans-serif;--wp--preset--font-family--work-sans: 'Work Sans', sans-serif;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-color{color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-color{color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-color{color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-color{color: var(--wp--preset--color--light-gray) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-background-color{background-color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-background-color{background-color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-background-color{background-color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-background-color{background-color: var(--wp--preset--color--light-gray) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-border-color{border-color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-border-color{border-color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-border-color{border-color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-border-color{border-color: var(--wp--preset--color--light-gray) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}.has-albert-sans-font-family{font-family: var(--wp--preset--font-family--albert-sans) !important;}.has-alegreya-font-family{font-family: var(--wp--preset--font-family--alegreya) !important;}.has-arvo-font-family{font-family: var(--wp--preset--font-family--arvo) !important;}.has-bodoni-moda-font-family{font-family: var(--wp--preset--font-family--bodoni-moda) !important;}.has-bricolage-grotesque-font-family{font-family: var(--wp--preset--font-family--bricolage-grotesque) !important;}.has-cabin-font-family{font-family: var(--wp--preset--font-family--cabin) !important;}.has-chivo-font-family{font-family: var(--wp--preset--font-family--chivo) !important;}.has-commissioner-font-family{font-family: var(--wp--preset--font-family--commissioner) !important;}.has-cormorant-font-family{font-family: var(--wp--preset--font-family--cormorant) !important;}.has-courier-prime-font-family{font-family: var(--wp--preset--font-family--courier-prime) !important;}.has-crimson-pro-font-family{font-family: var(--wp--preset--font-family--crimson-pro) !important;}.has-dm-mono-font-family{font-family: var(--wp--preset--font-family--dm-mono) !important;}.has-dm-sans-font-family{font-family: var(--wp--preset--font-family--dm-sans) !important;}.has-dm-serif-display-font-family{font-family: var(--wp--preset--font-family--dm-serif-display) !important;}.has-domine-font-family{font-family: var(--wp--preset--font-family--domine) !important;}.has-eb-garamond-font-family{font-family: var(--wp--preset--font-family--eb-garamond) !important;}.has-epilogue-font-family{font-family: var(--wp--preset--font-family--epilogue) !important;}.has-fahkwang-font-family{font-family: var(--wp--preset--font-family--fahkwang) !important;}.has-figtree-font-family{font-family: var(--wp--preset--font-family--figtree) !important;}.has-fira-sans-font-family{font-family: var(--wp--preset--font-family--fira-sans) !important;}.has-fjalla-one-font-family{font-family: var(--wp--preset--font-family--fjalla-one) !important;}.has-fraunces-font-family{font-family: var(--wp--preset--font-family--fraunces) !important;}.has-gabarito-font-family{font-family: var(--wp--preset--font-family--gabarito) !important;}.has-ibm-plex-mono-font-family{font-family: var(--wp--preset--font-family--ibm-plex-mono) !important;}.has-ibm-plex-sans-font-family{font-family: var(--wp--preset--font-family--ibm-plex-sans) !important;}.has-ibarra-real-nova-font-family{font-family: var(--wp--preset--font-family--ibarra-real-nova) !important;}.has-instrument-serif-font-family{font-family: var(--wp--preset--font-family--instrument-serif) !important;}.has-inter-font-family{font-family: var(--wp--preset--font-family--inter) !important;}.has-josefin-sans-font-family{font-family: var(--wp--preset--font-family--josefin-sans) !important;}.has-jost-font-family{font-family: var(--wp--preset--font-family--jost) !important;}.has-libre-baskerville-font-family{font-family: var(--wp--preset--font-family--libre-baskerville) !important;}.has-libre-franklin-font-family{font-family: var(--wp--preset--font-family--libre-franklin) !important;}.has-literata-font-family{font-family: var(--wp--preset--font-family--literata) !important;}.has-lora-font-family{font-family: var(--wp--preset--font-family--lora) !important;}.has-merriweather-font-family{font-family: var(--wp--preset--font-family--merriweather) !important;}.has-montserrat-font-family{font-family: var(--wp--preset--font-family--montserrat) !important;}.has-newsreader-font-family{font-family: var(--wp--preset--font-family--newsreader) !important;}.has-noto-sans-mono-font-family{font-family: var(--wp--preset--font-family--noto-sans-mono) !important;}.has-nunito-font-family{font-family: var(--wp--preset--font-family--nunito) !important;}.has-open-sans-font-family{font-family: var(--wp--preset--font-family--open-sans) !important;}.has-overpass-font-family{font-family: var(--wp--preset--font-family--overpass) !important;}.has-pt-serif-font-family{font-family: var(--wp--preset--font-family--pt-serif) !important;}.has-petrona-font-family{font-family: var(--wp--preset--font-family--petrona) !important;}.has-piazzolla-font-family{font-family: var(--wp--preset--font-family--piazzolla) !important;}.has-playfair-display-font-family{font-family: var(--wp--preset--font-family--playfair-display) !important;}.has-plus-jakarta-sans-font-family{font-family: var(--wp--preset--font-family--plus-jakarta-sans) !important;}.has-poppins-font-family{font-family: var(--wp--preset--font-family--poppins) !important;}.has-raleway-font-family{font-family: var(--wp--preset--font-family--raleway) !important;}.has-roboto-font-family{font-family: var(--wp--preset--font-family--roboto) !important;}.has-roboto-slab-font-family{font-family: var(--wp--preset--font-family--roboto-slab) !important;}.has-rubik-font-family{font-family: var(--wp--preset--font-family--rubik) !important;}.has-rufina-font-family{font-family: var(--wp--preset--font-family--rufina) !important;}.has-sora-font-family{font-family: var(--wp--preset--font-family--sora) !important;}.has-source-sans-3-font-family{font-family: var(--wp--preset--font-family--source-sans-3) !important;}.has-source-serif-4-font-family{font-family: var(--wp--preset--font-family--source-serif-4) !important;}.has-space-mono-font-family{font-family: var(--wp--preset--font-family--space-mono) !important;}.has-syne-font-family{font-family: var(--wp--preset--font-family--syne) !important;}.has-texturina-font-family{font-family: var(--wp--preset--font-family--texturina) !important;}.has-urbanist-font-family{font-family: var(--wp--preset--font-family--urbanist) !important;}.has-work-sans-font-family{font-family: var(--wp--preset--font-family--work-sans) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-14-1' href='/_static/??/wp-content/themes/pub/penscratch-2/css/reset.css,/wp-content/themes/pub/penscratch-2/style.css?m=1779214100j&cssminify=yes' type='text/css' media='all' />
<style id="penscratch-2-style-inline-css">
.cat-links, .byline .author, .group-blog .byline + .sep, .single .byline + .sep { clip: rect(1px, 1px, 1px, 1px); height: 1px; position: absolute; overflow: hidden; width: 1px; }
/*# sourceURL=penscratch-2-style-inline-css */
</style>
<link rel='stylesheet' id='penscratch-2-fonts-css' href='https://fonts-api.wp.com/css?family=Roboto+Slab%3A300%2C400%2C700&#038;subset=latin%2Clatin-ext' media='all' />
<link crossorigin='anonymous' rel='stylesheet' id='all-css-16-1' href='/_static/??-eJyNjsEKAjEMRH/ImO0iLB7Eb+mG2K22aWmyLP691YuIIN7mHd7M4FaBihiLoS2cWbGuM1YWpeaNFhiRVHFOhW6673GHfyhRCNXuiWGrVPKXl1eoaQ1RFBv37tBjeA298ZcUuEB/5C0W+QC4JB/bUz3nk5um4+gObhiuD4S6VWY=&cssminify=yes' type='text/css' media='all' />
<link crossorigin='anonymous' rel='stylesheet' id='print-css-17-1' href='/wp-content/mu-plugins/global-print/global-print.css?m=1465851035i&cssminify=yes' type='text/css' media='print' />
<style id="jetpack-global-styles-frontend-style-inline-css">
:root { --font-headings: unset; --font-base: unset; --font-headings-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif; --font-base-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;}
/*# sourceURL=jetpack-global-styles-frontend-style-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-20-1' href='/_static/??-eJyNjcEKAjEMRH/ImhVlqwfxU6TbhrZrmhTTIv69u+JFvHgZ5sHwBh7VeOGG3KB0U6nHzAoztur87cNQRNYInVBBk7tjcCE83zVz3HrVDfxvumb2MPVMAVR8dmRIougX/DhbwrK8pwNEksnROriU884e9+PJDnaYXxwFS20=&cssminify=yes' type='text/css' media='all' />
<script id="jetpack_related-posts-js-extra">
var related_posts_js_options = {"post_heading":"h4"};
//# sourceURL=jetpack_related-posts-js-extra
</script>
<script id="wpcom-actionbar-placeholder-js-extra">
var actionbardata = {"siteID":"23592848","postID":"1235","siteURL":"https://blog.acolyer.org","xhrURL":"https://blog.acolyer.org/wp-admin/admin-ajax.php","nonce":"1fb13c386d","isLoggedIn":"","statusMessage":"","subsEmailDefault":"instantly","proxyScriptUrl":"https://s0.wp.com/wp-content/js/wpcom-proxy-request.js?m=1513050504i&amp;ver=20211021","shortlink":"https://wp.me/p1AZzO-jV","i18n":{"followedText":"New posts from this site will now appear in your \u003Ca href=\"https://wordpress.com/reader\"\u003EReader\u003C/a\u003E","foldBar":"Collapse this bar","unfoldBar":"Expand this bar","shortLinkCopied":"Shortlink copied to clipboard."}};
//# sourceURL=wpcom-actionbar-placeholder-js-extra
</script>
<script id="jetpack-mu-wpcom-settings-js-before">
var JETPACK_MU_WPCOM_SETTINGS = {"assetsUrl":"https://s1.wp.com/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/moon/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/"};
//# sourceURL=jetpack-mu-wpcom-settings-js-before
</script>
<script crossorigin='anonymous' type='text/javascript' src='/_static/??/wp-content/mu-plugins/jetpack-plugin/moon/_inc/build/related-posts/related-posts.min.js,/wp-content/js/rlt-proxy.js?m=1788798821j'></script>
<script id="rlt-proxy-js-after">
rltInitialize( {"token":null,"iframeOrigins":["https:\/\/widgets.wp.com"]} );
//# sourceURL=rlt-proxy-js-after
</script>
<link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://adriancolyer.wordpress.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress.com" />
<link rel="canonical" href="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/" />
<link rel='shortlink' href='https://wp.me/p1AZzO-jV' />
<link rel="alternate" type="application/json+oembed" href="https://public-api.wordpress.com/oembed/?format=json&amp;url=https%3A%2F%2Fblog.acolyer.org%2F2015%2F10%2F16%2Fholistic-configuration-management-at-facebook%2F&amp;for=wpcom-auto-discovery" /><link rel="alternate" type="application/xml+oembed" href="https://public-api.wordpress.com/oembed/?format=xml&amp;url=https%3A%2F%2Fblog.acolyer.org%2F2015%2F10%2F16%2Fholistic-configuration-management-at-facebook%2F&amp;for=wpcom-auto-discovery" />
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="Holistic Configuration Management at Facebook" />
<meta property="og:url" content="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/" />
<meta property="og:description" content="Holistic Configuration Management at Facebook &#8211; Tang et al. (Facebook) 2015 This paper gives a comprehensive description of the use cases, design, implementation, and usage statistics of a su…" />
<meta property="article:published_time" content="2015-10-16T05:00:00+00:00" />
<meta property="article:modified_time" content="2017-07-27T07:42:12+00:00" />
<meta property="og:site_name" content="the morning paper" />
<meta property="og:image" content="https://blog.acolyer.org/wp-content/uploads/2015/10/facebook-config.png?w=426" />
<meta property="og:image:width" content="426" />
<meta property="og:image:height" content="1024" />
<meta property="og:image:alt" content="Configuration flows at Facebook" />
<meta property="og:locale" content="en_US" />
<meta property="article:publisher" content="https://www.facebook.com/WordPresscom" />
<meta name="twitter:text:title" content="Holistic Configuration Management at&nbsp;Facebook" />
<meta name="twitter:image" content="https://blog.acolyer.org/wp-content/uploads/2015/10/facebook-config.png?w=640" />
<meta name="twitter:image:alt" content="Configuration flows at Facebook" />
<meta name="twitter:card" content="summary_large_image" />
<!-- End Jetpack Open Graph Tags -->
<link rel="shortcut icon" type="image/x-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=32" sizes="16x16" />
<link rel="icon" type="image/x-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=32" sizes="16x16" />
<link rel="apple-touch-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=114" />
<link rel='openid.server' href='https://blog.acolyer.org/?openidserver=1' />
<link rel='openid.delegate' href='https://blog.acolyer.org/' />
<link rel="search" type="application/opensearchdescription+xml" href="https://blog.acolyer.org/osd.xml" title="the morning paper" />
<link rel="search" type="application/opensearchdescription+xml" href="https://s1.wp.com/opensearch.xml" title="WordPress.com" />
<meta name="theme-color" content="#ffffff" />
<meta name="description" content="Holistic Configuration Management at Facebook - Tang et al. (Facebook) 2015 This paper gives a comprehensive description of the use cases, design, implementation, and usage statistics of a suite of tools that manage Facebook’s configuration end-to-end, including the frontend products, backend systems, and mobile apps. The configuration for Facebook&#039;s site is updated thousands of times&hellip;" />
<style id="custom-background-css">
body.custom-background { background-color: #ffffff; }
</style>
<style type="text/css" id="custom-colors-css"> .widget a:hover,
.site-footer .widget a:hover {
color: #666;
}
.jetpack-social-navigation ul a:hover { color: #FFFFFF;}
body { background-color: #ffffff;}
.jetpack-social-navigation ul a:hover { background-color: #1c7c7c;}
.jetpack-recipe-title { border-color: #1c7c7c;}
.toggled .menu-toggle, .wp_widget_tag_cloud a:hover, .tags-links a:hover { border-color: #1B7878;}
.button:hover, .button:focus, .button:active, button:hover, button:focus, button:active, input[type="button"]:hover, input[type="button"]:focus, input[type="button"]:active, input[type="reset"]:hover, input[type="reset"]:focus, input[type="reset"]:active, input[type="submit"]:hover, input[type="submit"]:focus, input[type="submit"]:active, #infinite-handle span:hover { border-color: #1B7878;}
.button:hover, .button:focus, .button:active, button:hover, button:focus, button:active, input[type="button"]:hover, input[type="button"]:focus, input[type="button"]:active, input[type="reset"]:hover, input[type="reset"]:focus, input[type="reset"]:active, input[type="submit"]:hover, input[type="submit"]:focus, input[type="submit"]:active, #infinite-handle span:hover { color: #1B7878;}
.menu-toggle:hover #menu-icon, .toggled #menu-icon { fill: #1B7878;}
a, a:visited, .site-footer a:hover, .main-navigation li:hover > a, .current_page_item a, .current-menu-item a, .toggled .menu-toggle, .entry-footer a:hover, .entry-meta a:hover, .entry-title a:hover, .site-main .post-navigation .meta-nav, .site-main .paging-navigation .meta-nav, .comment-meta .comment-metadata a:hover, .required, .widget a, #wp-calendar a, .wp_widget_tag_cloud a:hover, #infinite-footer .container a:hover, #infinite-handle span, .tags-links a:hover, .current_page_item a, .current-menu-item a, .current_page_item a:visited, .current-menu-item a:visited { color: #1B7878;}
</style>
<link rel="stylesheet" id="custom-css-css" type="text/css" href="https://s2.wp.com/?custom-css=1&#038;csblog=1AZzO&#038;cscache=6&#038;csrev=16" />
<link crossorigin='anonymous' rel='stylesheet' id='all-css-0-3' href='/_static/??-eJyNjMEKhDAMBX9IfaiIXsRPWdxskGqbFNPi7y+CevY4wzA4YkkqiSUh5DL6vDgxrJziTNvFCKqCjxPC1yttBjtc5L0iswLvD0F/2bOB5l2zsX+iW5zDKYx1P3RN07b1sP4BKPs7aw==&cssminify=yes' type='text/css' media='all' />
</head>
<body class="wp-singular post-template-default single single-post postid-1235 single-format-standard custom-background wp-embed-responsive wp-theme-pubpenscratch-2 customizer-styles-applied no-sidebar jetpack-reblog-enabled categories-hidden author-hidden custom-colors">
<div id="page" class="hfeed site">
<a class="skip-link screen-reader-text" href="#content">Skip to content</a>
<header id="masthead" class="site-header" role="banner">
<div class="site-branding">
<p class="site-title"><a href="https://blog.acolyer.org/" rel="home">the morning paper</a></p>
<p class="site-description">a random walk through Computer Science research, by Adrian Colyer</p>
</div>
<nav id="site-navigation" class="main-navigation" role="navigation">
<button class="menu-toggle">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><rect x="0" fill="none" width="16" height="16"/><g><path id="menu-icon" d="M0 14h16v-2H0v2zM0 2v2h16V2H0zm0 7h16V7H0v2z"/></g></svg>
Menu </button>
<div class="menu"><ul>
<li class="page_item page-item-2"><a href="https://blog.acolyer.org/about/">About</a></li>
<li class="page_item page-item-8716"><a href="https://blog.acolyer.org/archives/">Archives</a></li>
<li class="page_item page-item-5027"><a href="https://blog.acolyer.org/infoq-quarterly-review-editions/">InfoQ QR Editions</a></li>
<li class="page_item page-item-8713"><a href="https://blog.acolyer.org/search/">Search</a></li>
<li class="page_item page-item-406"><a href="https://blog.acolyer.org/email-subs/">Subscribe</a></li>
<li class="page_item page-item-8719"><a href="https://blog.acolyer.org/tags/">Tags</a></li>
<li class="page_item page-item-6991"><a href="https://blog.acolyer.org/privacy/">Privacy</a></li>
</ul></div>
</nav><!-- #site-navigation -->
</header><!-- #masthead -->
<div id="content" class="site-content">
<div id="primary" class="content-area">
<main id="main" class="site-main" role="main">
<article id="post-1235" class="post-1235 post type-post status-publish format-standard hentry category-uncategorized tag-facebook tag-operations tag-testing">
<header class="entry-header">
<h1 class="entry-title">Holistic Configuration Management at&nbsp;Facebook</h1>
<div class="entry-meta">
<span class="posted-on"><a href="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/" rel="bookmark"><time class="entry-date published" datetime="2015-10-16T06:00:00+01:00">October 16, 2015</time><time class="updated" datetime="2017-07-27T08:42:12+01:00">July 27, 2017</time></a></span><span class="byline"><span class="author vcard"><span class="sep"> ~ </span><a class="url fn n" href="https://blog.acolyer.org/author/adriancolyer/">adriancolyer</a></span></span> </div><!-- .entry-meta -->
</header><!-- .entry-header -->
<div class="entry-content">
<p><a href="http://sigops.org/sosp/sosp15/current/2015-Monterey/printable/008-tang.pdf">Holistic Configuration Management at Facebook</a> &#8211; Tang et al. (Facebook) 2015</p>
<blockquote><p>
This paper gives a comprehensive description of the use cases, design, implementation, and usage statistics of a suite of tools that manage Facebook’s configuration end-to-end, including the frontend products, backend systems, and mobile apps.
</p></blockquote>
<p>The configuration for Facebook&#8217;s site is updated thousands of times per day. There are also thousands of engineers who make those configuration changes. Thousands of changes made by thousands of people is a recipe for configuration errors &#8211; a major source of site outages. Therefore Facebook need a holistic configuration solution that is robust and can cope with this rate of change. One of the reasons that Facebook make so many configuration changes, is that they use configuration so for many different things:</p>
<ul>
<li>For controlling which users experience a new feature &#8211; for example, whether 1%, 5% or 10% of users should see it is managed via configuration</li>
<li>For conducting live experiments (e.g. A/B tests) to try out different parameters</li>
<li>For application-level traffic control&#8230;</li>
</ul>
<blockquote><p>
Configs help manage the site’s traffic in many ways. Automation tools periodically make config changes to shift traffic across regions and perform load tests in production. In case of emergency, a config change kicks off automated cluster/region traffic drain and another config change disables resource-hungry features of the site. During shadow tests, a config change starts or stops duplicating live traffic to testing servers. During a fire drill, a config change triggers fault injection into a production system to evaluate its resilience.
</p></blockquote>
<ul>
<li>For topology setup and load-balancing of <a href="https://blog.acolyer.org/2015/05/19/tao-facebooks-distributed-data-store-for-the-social-graph/">TAO</a></li>
<li>For control of the monitoring stack:</li>
</ul>
<blockquote><p>
Facebook’s monitoring stack is controlled through config changes: 1) what monitoring data to collect, 2) monitoring dashboard (e.g., the layout of the key-metric graphs), 3) alert detection rules (i.e., what is considered an anomaly), 4) alert subscription rules (i.e., who should be paged), and 5) automated remediation actions [27], e.g., rebooting or reimaging a server. All these can be dynamically changed without a code upgrade, e.g., as troubleshooting requires collecting more monitoring data.
</p></blockquote>
<ul>
<li>For updating machine learning models</li>
<li>To control an application&#8217;s internal behaviour (i.e., the &#8216;classic&#8217; configuration use case)</li>
</ul>
<p><em>All of these</em> are handled via configuration which is canonically stored in git with a watcher that monitors the git repositories and pushes changes to live systems. Sweet!</p>
<p>The config itself is often generated by code, with a defined set of validators for config values also provided in code:</p>
<blockquote><p>
We take a truly configuration-as-code approach to compile and generate configs from high-level source code. We store the config programs and the generated configs in a version control tool.
</p></blockquote>
<p>Before a configuration change makes it into production there are four layers of safeguards:</p>
<blockquote><p>
First, the configuration compiler automatically runs validators to verify invariants defined for configs. Second, a config change is treated the same as a code change and goes though the same rigorous code review process. Third, a config change that affects the frontend products automatically goes through continuous integration tests in a sandbox. Lastly, the automated canary testing tool rolls out a config change to production in a staged fashion.
</p></blockquote>
<p>Canary testing itself is also driven through configuration:</p>
<blockquote><p>
A config is associated with a canary spec that describes how to automate testing the config in production. The spec defines multiple testing phases. For example, in phase 1, test on 20 servers; in phase 2, test in a full cluster with thousands of servers. For each phase, it specifies the testing target servers, the healthcheck metrics, and the predicates that decide whether the test passes or fails. For example, the click-through rate (CTR) collected from the servers using the new config should not be more than x% lower than the CTR collected from the servers still using the old config.
</p></blockquote>
<p>At the core of Facebook&#8217;s configuration management process is a tool called the configerator [sic]. It provides all of the foundational capabilities: version control, authoring, code review, automated canary testing, and <em>config distribution</em>.</p>
<blockquote><p>
Configerator distributes a config update to hundreds of thousands of servers scattered across multiple continents. In such an environment, failures are the norm. In addition to scalability and reliability, other properties important to Configerator are 1) availability (i.e., an application should continue to run regardless of failures in the configuration management tools); and 2) data consistency (i.e., an application’s instances running on different servers should eventually receive all config updates delivered in the same order, although there is no guarantee that they all receive a config update exactly at the same time).
</p></blockquote>
<p>A <em>Git Tailer</em> continuously monitors git for changes and pushes them to Zeus (a forked version of ZooKeeper) for distribution. Zeus uses a three-level, high fan-out distribution tree to distribute configs using a push model. There are <em>leaders</em>, <em>observers</em>, and <em>proxies</em>. Each data center has multiple clusters, each with thousands of servers. A Zeus leader first commits a write on its followers, and then asynchronously pushes the write to observers (of which it may have hundreds). Observers keep a fully replicated read-only copy of the leader&#8217;s data. Every server runs a configuration proxy, which randomly picks an observer to connect to. Applications use the proxy to fetch their config:</p>
<blockquote><p>
The proxy reads the config from an observer with a watch so that later the observer will notify the proxy if the config is updated. The proxy stores the config in an on-disk cache for later reuse. If the proxy fails, the application falls back to read from the on-disk cache directly. This design provides high availability. So long as a config exists in the on-disk cache, the application can access it (though outdated), even if all Configerator components fail, including the git repository, Zeus leader/followers, observers, and proxy.
</p></blockquote>
<p>For large configuration objects, a separate peer-to-peer distribution mechanism is used.</p>
<p>The full end-to-end flow is illustrated by this diagram:<br />
<a href="https://blog.acolyer.org/wp-content/uploads/2015/10/facebook-config.png"><img data-attachment-id="1236" data-permalink="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/facebook-config/" data-orig-file="https://blog.acolyer.org/wp-content/uploads/2015/10/facebook-config.png" data-orig-size="461,1108" data-comments-opened="1" data-image-meta="{&quot;aperture&quot;:&quot;0&quot;,&quot;credit&quot;:&quot;&quot;,&quot;camera&quot;:&quot;&quot;,&quot;caption&quot;:&quot;&quot;,&quot;created_timestamp&quot;:&quot;0&quot;,&quot;copyright&quot;:&quot;&quot;,&quot;focal_length&quot;:&quot;0&quot;,&quot;iso&quot;:&quot;0&quot;,&quot;shutter_speed&quot;:&quot;0&quot;,&quot;title&quot;:&quot;&quot;,&quot;orientation&quot;:&quot;0&quot;}" data-image-title="Facebook Config" data-image-description="" data-image-caption="" data-large-file="https://blog.acolyer.org/wp-content/uploads/2015/10/facebook-config.png?w=426" src="https://blog.acolyer.org/wp-content/uploads/2015/10/facebook-config.png?w=426&#038;h=1024" alt="Configuration flows at Facebook" width="426" height="1024" class="aligncenter size-large wp-image-1236" srcset="https://blog.acolyer.org/wp-content/uploads/2015/10/facebook-config.png?w=426 426w, https://blog.acolyer.org/wp-content/uploads/2015/10/facebook-config.png?w=62 62w, https://blog.acolyer.org/wp-content/uploads/2015/10/facebook-config.png 461w" sizes="(max-width: 426px) 100vw, 426px" /></a></p>
<p>The &#8216;landing strip&#8217; is there to address contention caused by so many engineers making concurrent commits into a shared config repository.</p>
<blockquote><p>
The “Landing Strip” in Figure 3 alleviates the problem, by 1) receiving diffs from committers, 2) serializing them according to the first-come-first-served order, and 3) pushing them to the shared git repository on behalf of the committers, without requiring the committers to bring their local repository clones up to date. If there is a true conflict between a diff being pushed and some previously committed diffs, the shared git repository rejects the diff, and the error is relayed back to the committer. Only then, the committer has to update her local repository clone and resolve the conflict.
</p></blockquote>
<p>This helps but does not fully alleviate the problem. The issue arise because all configuration for all projects is stored in one git repository. The Facebook team are investigating a configuration tree approach where different git repositories are used for different branches of the tree (e.g. /feed and /tao).</p>
<p>On top of the base configuration infrastructure runs the <em>Gatekeeper</em>. Gatekeeper manages code rollout through online config changes:</p>
<blockquote><p>
While a new product feature is still under development, Facebook engineers commonly release the new code into production early but in a disabled mode, and then use Gatekeeper to incrementally enable it online. If any problem is detected during the rollout, the new code can be disabled instantaneously. Without changing any source code, a typical launch using Gatekeeper goes through multiple phases. For example, initially Gatekeeper may only enable the product feature to the engineers developing the feature. Then Gatekeeper can enable the feature for an increasing percentage of Facebook employees, e.g., 1%→10%→100%. After successful internal testing, it can target 5% of the users from a specific region. Finally, the feature can be launched globally with an increasing coverage, e.g., 1%→10%→100%.
</p></blockquote>
<p>To enable or disable a product feature, Gatekeeper first evaluates a set of configuration-specified constraints called <em>restraints</em>. If these restraints are satisfied, it probabilistically determines whether to pass or fail the gate depending on a configurable property that controls user sampling (e.g. 1% or 10%).</p>
<blockquote><p>
Examples of restraints include checking whether the user is a Facebook employee and checking the type of a mobile device&#8230; A Gatekeeper project is dynamically composed out of restraints through configuration. Internally, a restraint is statically implemented in PHP or C++. Currently, hundreds of restraints have been implemented, which are used to compose tens of thousands of Gatekeeper projects. The restraints check various conditions of a user, e.g., country/region, locale, mobile app, device, new user, and number of friends.
</p></blockquote>
<p>All of this control logic is stored as a config that can be changed live without a code update.</p>
<blockquote><p>
Gatekeeper projects manage product feature rollouts. When a user accesses facebook.com, the Gatekeeper projects are checked in realtime to determine what features to enable for the user. Because the check throughput is high (billions of checks per second) and some Gatekeeper restraints are data intensive, currently Gatekeeper consumes a significant percentage of the total CPU of the frontend clusters that consist of hundreds of thousands of servers. We constantly work on improving Gatekeeper’s efficiency. On the other hand, we consider this “overhead” worthwhile, because it enables Facebook engineers to iterate rapidly on new product features. This is evidenced by the popularity of Gatekeeper. In 2014, tens of thousands of Gatekeeper projects were created or updated to actively manage the rollouts of a huge number of micro product features.
</p></blockquote>
<p>When an engineer makes a change, it takes about 10 minutes to go through the canary process, 5 seconds to then commit the change into the shared git repository, another 5 seconds for the tailer to fetch the change from the repository and write it to Zeus, and about 4.5 seconds from there to reach hundreds of thousands of servers across multiple continents.</p>
<p>Is all of this machinery something that only applies at Facebook scale? For the general principles at least, no &#8211; anyone trying to continuously deliver new features into production can benefit. The authors conclude the paper by contemplating what lessons and principles might be applicable outside of Facebook.</p>
<blockquote><p>
The technology we described is not exclusive for large Internet services. It matters for small systems as well.. .Agile configuration management enables agile software development.
</p></blockquote>
<p>Among many other lessons, I like this one:</p>
<blockquote><p>
Although the use cases of configuration management can be very diverse (e.g., from gating product rollouts to A/B testing), it is feasible and beneficial to support all of them on top of a uniform and flexible foundation, with additional tools providing specialized functions. Otherwise, inferior wheels will be reinvented. At Facebook, it is a long history of fragmented solutions converging onto Configerator.
</p></blockquote>
<div id="jp-post-flair" class="sharedaddy sd-like-enabled sd-sharing-enabled"><div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-twitter"><a rel="nofollow noopener noreferrer"
data-shared="sharing-twitter-1235"
class="share-twitter sd-button share-icon"
href="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/?share=twitter"
target="_blank"
aria-labelledby="sharing-twitter-1235"
>
<span id="sharing-twitter-1235" hidden>Share on X (Opens in new window)</span>
<span>X</span>
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
data-shared="sharing-linkedin-1235"
class="share-linkedin sd-button share-icon"
href="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/?share=linkedin"
target="_blank"
aria-labelledby="sharing-linkedin-1235"
>
<span id="sharing-linkedin-1235" hidden>Share on LinkedIn (Opens in new window)</span>
<span>LinkedIn</span>
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
data-shared="sharing-email-1235"
class="share-email sd-button share-icon"
href="mailto:?subject=%5BShared%20Post%5D%20Holistic%20Configuration%20Management%20at%20Facebook&#038;body=https%3A%2F%2Fblog.acolyer.org%2F2015%2F10%2F16%2Fholistic-configuration-management-at-facebook%2F&#038;share=email"
target="_blank"
aria-labelledby="sharing-email-1235"
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you&#039;re having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="61a9818dbe" data-email-share-track-url="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/?share=email">
<span id="sharing-email-1235" hidden>Email a link to a friend (Opens in new window)</span>
<span>Email</span>
</a></li><li class="share-print"><a rel="nofollow noopener noreferrer"
data-shared="sharing-print-1235"
class="share-print sd-button share-icon"
href="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/#print?share=print"
target="_blank"
aria-labelledby="sharing-print-1235"
>
<span id="sharing-print-1235" hidden>Print (Opens in new window)</span>
<span>Print</span>
</a></li><li class="share-end"></li></ul></div></div></div><div class='sharedaddy sd-block sd-like jetpack-likes-widget-wrapper jetpack-likes-widget-unloaded' id='like-post-wrapper-23592848-1235-6aa362731e025' data-src='//widgets.wp.com/likes/index.html?ver=20260911#blog_id=23592848&amp;post_id=1235&amp;origin=adriancolyer.wordpress.com&amp;obj_id=23592848-1235-6aa362731e025&amp;domain=blog.acolyer.org' data-name='like-post-frame-23592848-1235-6aa362731e025' data-title='Like or Reblog'><div class='likes-widget-placeholder post-likes-widget-placeholder' style='height: 55px;'><span class='button'><span>Like</span></span> <span class='loading'>Loading...</span></div><span class='sd-text-color'></span><a class='sd-link-color'></a></div>
<div id='jp-relatedposts' class='jp-relatedposts' >
<h3 class="jp-relatedposts-headline"><em>Related</em></h3>
</div></div> </div><!-- .entry-content -->
<footer class="entry-footer">
<span class="cat-links">
Posted in <a href="https://blog.acolyer.org/category/uncategorized/" rel="category tag">Uncategorized</a> </span>
<span class="tags-links">
<a href="https://blog.acolyer.org/tag/facebook/" rel="tag">Facebook</a><a href="https://blog.acolyer.org/tag/operations/" rel="tag">Operations</a><a href="https://blog.acolyer.org/tag/testing/" rel="tag">Testing</a> </span>
</footer><!-- .entry-footer -->
</article><!-- #post-## -->
<nav class="navigation post-navigation" aria-label="Posts">
<h2 class="screen-reader-text">Post navigation</h2>
<div class="nav-links"><div class="nav-previous"><a href="https://blog.acolyer.org/2015/10/15/ironfleet-proving-practical-distributed-systems-correc/" rel="prev"><span class="meta-nav">&lsaquo; Previous</span>IronFleet: Proving Practical Distributed Systems&nbsp;Correct</a></div><div class="nav-next"><a href="https://blog.acolyer.org/2015/10/19/existential-consistency-measuring-and-understanding-consistency-at-facebook/" rel="next"><span class="meta-nav">Next &rsaquo;</span>Existential Consistency: Measuring and Understanding Consistency at&nbsp;Facebook</a></div></div>
</nav>
<div id="comments" class="comments-area">
<h2 class="comments-title">
14 thoughts on &ldquo;<span>Holistic Configuration Management at&nbsp;Facebook</span>&rdquo; </h2><!-- .comments-title -->
<ol class="comment-list">
<li id="comment-3949" class="pingback even thread-even depth-1">
<div class="comment-body">
Pingback: <a href="https://highops.com/insights/top-10-links-for-the-week-of-oct-12/" class="url" rel="ugc external nofollow">Top 10 links for the week of Oct 12 - HighOps</a> </div>
</li><!-- #comment-## -->
<li id="comment-4174" class="pingback odd alt thread-odd thread-alt depth-1">
<div class="comment-body">
Pingback: <a href="https://highops.com/insights/top-10-links-for-the-month-of-oct-week-of-oct-26/" class="url" rel="ugc external nofollow">Top 10 links for the month of Oct + week of Oct 26 - HighOps</a> </div>
</li><!-- #comment-## -->
<li id="comment-5311" class="pingback even thread-even depth-1">
<div class="comment-body">
Pingback: <a href="https://blog.acolyer.org/2015/12/14/a-year-in-papers/" class="url" rel="ugc">A Year in Papers | the morning paper</a> </div>
</li><!-- #comment-## -->
<li id="comment-5632" class="comment odd alt thread-odd thread-alt depth-1 parent">
<article id="div-comment-5632" class="comment-body">
<footer class="comment-meta">
<div class="comment-author vcard">
<img referrerpolicy="no-referrer" alt='Andrew&#039;s avatar' src='https://0.gravatar.com/avatar/064bde5432620c6a6978d60c731a506d7ebc67f020fdc988a4e9ec4818344874?s=60&#038;d=identicon&#038;r=G' srcset='https://0.gravatar.com/avatar/064bde5432620c6a6978d60c731a506d7ebc67f020fdc988a4e9ec4818344874?s=60&#038;d=identicon&#038;r=G 1x, https://0.gravatar.com/avatar/064bde5432620c6a6978d60c731a506d7ebc67f020fdc988a4e9ec4818344874?s=90&#038;d=identicon&#038;r=G 1.5x, https://0.gravatar.com/avatar/064bde5432620c6a6978d60c731a506d7ebc67f020fdc988a4e9ec4818344874?s=120&#038;d=identicon&#038;r=G 2x, https://0.gravatar.com/avatar/064bde5432620c6a6978d60c731a506d7ebc67f020fdc988a4e9ec4818344874?s=180&#038;d=identicon&#038;r=G 3x, https://0.gravatar.com/avatar/064bde5432620c6a6978d60c731a506d7ebc67f020fdc988a4e9ec4818344874?s=240&#038;d=identicon&#038;r=G 4x' class='avatar avatar-60' height='60' width='60' decoding='async' /> <b class="fn">Andrew</b> <span class="says">says:</span> </div><!-- .comment-author -->
<div class="comment-metadata">
<a href="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/#comment-5632"><time datetime="2015-12-30T01:35:36+00:00">December 30, 2015 at 1:35 am</time></a> </div><!-- .comment-metadata -->
</footer><!-- .comment-meta -->
<div class="comment-content">
<p>&#8220;Every server runs a configuration proxy, which randomly picks an observer to connect to. Applications use the proxy to fetch their config&#8221;.<br />
But at the same time they write &#8220;Configerator uses the push model&#8221;.</p>
<p>If every server has to connect to a proxy, it looks like the pull model, does not it?</p>
</div><!-- .comment-content -->
</article><!-- .comment-body -->
<ol class="children">
<li id="comment-5637" class="comment byuser comment-author-adriancolyer bypostauthor even depth-2">
<article id="div-comment-5637" class="comment-body">
<footer class="comment-meta">
<div class="comment-author vcard">
<img referrerpolicy="no-referrer" alt='adriancolyer&#039;s avatar' src='https://0.gravatar.com/avatar/3a354cf7a8268ff825f87b3c13e24831e4d773c637ee1a81895ed8e861fa8f1d?s=60&#038;d=identicon&#038;r=G' srcset='https://0.gravatar.com/avatar/3a354cf7a8268ff825f87b3c13e24831e4d773c637ee1a81895ed8e861fa8f1d?s=60&#038;d=identicon&#038;r=G 1x, https://0.gravatar.com/avatar/3a354cf7a8268ff825f87b3c13e24831e4d773c637ee1a81895ed8e861fa8f1d?s=90&#038;d=identicon&#038;r=G 1.5x, https://0.gravatar.com/avatar/3a354cf7a8268ff825f87b3c13e24831e4d773c637ee1a81895ed8e861fa8f1d?s=120&#038;d=identicon&#038;r=G 2x, https://0.gravatar.com/avatar/3a354cf7a8268ff825f87b3c13e24831e4d773c637ee1a81895ed8e861fa8f1d?s=180&#038;d=identicon&#038;r=G 3x, https://0.gravatar.com/avatar/3a354cf7a8268ff825f87b3c13e24831e4d773c637ee1a81895ed8e861fa8f1d?s=240&#038;d=identicon&#038;r=G 4x' class='avatar avatar-60' height='60' width='60' decoding='async' /> <b class="fn"><a href="https://adriancolyer.wordpress.com" class="url" rel="ugc external nofollow">adriancolyer</a></b> <span class="says">says:</span> </div><!-- .comment-author -->
<div class="comment-metadata">
<a href="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/#comment-5637"><time datetime="2015-12-30T08:52:27+00:00">December 30, 2015 at 8:52 am</time></a> </div><!-- .comment-metadata -->
</footer><!-- .comment-meta -->
<div class="comment-content">
<p>From my recollection, updates are pushed to the configuration proxies, and applications pull from there. So it&#8217;s a mixture of both modes&#8230;</p>
</div><!-- .comment-content -->
</article><!-- .comment-body -->
</li><!-- #comment-## -->
</ol><!-- .children -->
</li><!-- #comment-## -->
<li id="comment-5839" class="pingback odd alt thread-even depth-1">
<div class="comment-body">
Pingback: <a href="http://sreweekly.com/sre-weekly-issue-5/" class="url" rel="ugc external nofollow">SRE Weekly Issue #5 &#8211; SRE WEEKLY</a> </div>
</li><!-- #comment-## -->
<li id="comment-6053" class="pingback even thread-odd thread-alt depth-1">
<div class="comment-body">
Pingback: <a href="https://blog.acolyer.org/2016/01/24/a-distributed-systems-seminar-reading-list/" class="url" rel="ugc">A Distributed Systems Seminar Reading List&#8230; | the morning paper</a> </div>
</li><!-- #comment-## -->
<li id="comment-6506" class="pingback odd alt thread-even depth-1">
<div class="comment-body">
Pingback: <a href="https://blog.acolyer.org/2016/02/29/machine-learning-the-high-interest-credit-card-of-technical-debt/" class="url" rel="ugc">Machine Learning: The High-Interest Credit Card of Technical Debt | the morning paper</a> </div>
</li><!-- #comment-## -->
<li id="comment-6621" class="pingback even thread-odd thread-alt depth-1">
<div class="comment-body">
Pingback: <a href="https://www.cronweekly.com/issue-19/" class="url" rel="ugc external nofollow">issue #19: GUI&#039;s, Jenkins, Docker, Microsoft, jo, sshtron, cockpit &amp; more! - Cron Weekly: a weekly newsletter for Linux and Open Source sysadmins</a> </div>
</li><!-- #comment-## -->
<li id="comment-6710" class="pingback odd alt thread-even depth-1">
<div class="comment-body">
Pingback: <a href="https://sarahistestingblog.wordpress.com/2016/03/22/qcon/" class="url" rel="ugc external nofollow">QCon | sarahistesting</a> </div>
</li><!-- #comment-## -->
<li id="comment-6786" class="pingback even thread-odd thread-alt depth-1">
<div class="comment-body">
Pingback: <a href="https://smplnote.wordpress.com/2016/03/29/holistic-configuration-management-at-facebook-the-morning-paper/" class="url" rel="ugc external nofollow">Holistic Configuration Management at Facebook | the morning paper | simple note</a> </div>
</li><!-- #comment-## -->
<li id="comment-8855" class="pingback odd alt thread-even depth-1">
<div class="comment-body">
Pingback: <a href="https://blog.acolyer.org/2016/11/28/kraken-leveraging-live-traffic-tests-to-identify-and-resolve-resource-utilization-bottlenecks-in-large-scale-web-services/" class="url" rel="ugc">Kraken: Leveraging live traffic tests to identify and resolve resource utilization bottlenecks in large scale web services | the morning paper</a> </div>
</li><!-- #comment-## -->
<li id="comment-8864" class="pingback even thread-odd thread-alt depth-1">
<div class="comment-body">
Pingback: <a href="https://blog.acolyer.org/2016/11/29/early-detection-of-configuration-errors-to-reduce-failure-damage/" class="url" rel="ugc">Early detection of configuration errors to reduce failure damage | the morning paper</a> </div>
</li><!-- #comment-## -->
<li id="comment-18416" class="pingback odd alt thread-even depth-1">
<div class="comment-body">
Pingback: <a href="https://blog.acolyer.org/2017/11/22/canopy-an-end-to-end-performance-tracing-and-analysis-system/" class="url" rel="ugc">Canopy: an end-to-end performance tracing and analysis system | the morning paper</a> </div>
</li><!-- #comment-## -->
</ol><!-- .comment-list -->
<p class="no-comments">Comments are closed.</p>
</div><!-- #comments -->
</main><!-- #main -->
</div><!-- #primary -->
</div><!-- #content -->
<footer id="colophon" class="site-footer" role="contentinfo">
<div class="site-info-wrapper clear">
<div class="site-info">
<a href="https://wordpress.com/?ref=footer_blog" rel="nofollow">Blog at WordPress.com.</a>
</div><!-- .site-info -->
</div><!-- .site-info-wrapper -->
</footer><!-- #colophon -->
</div><!-- #page -->
<!-- -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/files/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/pub/penscratch-2/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script id="grofiles-cards-js" src="//0.gravatar.com/js/hovercards/hovercards.min.js?ver=202637d1133547e8a1f2801609f51cef12522abe7f6647d869e9fbc974fe356a37e613"></script>
<script id="wpgroho-js-extra">
var WPGroHo = {"my_hash":""};
//# sourceURL=wpgroho-js-extra
</script>
<script crossorigin='anonymous' type='text/javascript' src='/wp-content/mu-plugins/gravatar-hovercards/wpgroho.js?m=1610363240i'></script>
<script>
// Initialize and attach hovercards to all gravatars
( function() {
function init() {
if ( typeof Gravatar === 'undefined' ) {
return;
}
if ( typeof Gravatar.init !== 'function' ) {
return;
}
Gravatar.profile_cb = function ( hash, id ) {
WPGroHo.syncProfileData( hash, id );
};
Gravatar.my_hash = WPGroHo.my_hash;
Gravatar.init(
'body',
'#wp-admin-bar-my-account',
{
i18n: {
'Edit your profile →': 'Edit your profile →',
'View profile →': 'View profile →',
'Contact': 'Contact',
'Send money': 'Send money',
'Sorry, we are unable to load this Gravatar profile.': 'Sorry, we are unable to load this Gravatar profile.',
'Gravatar not found.': 'Gravatar not found.',
'Too Many Requests.': 'Too Many Requests.',
'Internal Server Error.': 'Internal Server Error.',
'Is this you?': 'Is this you?',
'Claim your free profile.': 'Claim your free profile.',
'Email': 'Email',
'Home Phone': 'Home Phone',
'Work Phone': 'Work Phone',
'Cell Phone': 'Cell Phone',
'Contact Form': 'Contact Form',
'Calendar': 'Calendar',
},
}
);
}
if ( document.readyState !== 'loading' ) {
init();
} else {
document.addEventListener( 'DOMContentLoaded', init );
}
} )();
</script>
<div style="display:none">
<div class="grofile-hash-map-b78ada3428560605c8807e47a266ee1c">
</div>
<div class="grofile-hash-map-a795b4f89a6d096f314fc0a2c80479c1">
</div>
</div>
<div id="actionbar" dir="ltr" style="display: none;"
class="actnbr-pub-penscratch-2 actnbr-has-follow actnbr-has-actions">
<ul>
<li class="actnbr-btn actnbr-hidden">
<a class="actnbr-action actnbr-actn-reblog" href="">
<svg class="gridicon gridicons-reblog" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M22.086 9.914L20 7.828V18c0 1.105-.895 2-2 2h-7v-2h7V7.828l-2.086 2.086L14.5 8.5 19 4l4.5 4.5-1.414 1.414zM6 16.172V6h7V4H6c-1.105 0-2 .895-2 2v10.172l-2.086-2.086L.5 15.5 5 20l4.5-4.5-1.414-1.414L6 16.172z"/></g></svg><span>Reblog</span>
</a>
</li>
<li class="actnbr-btn actnbr-hidden">
<a class="actnbr-action actnbr-actn-follow " href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
<span>Subscribe</span>
</a>
<a class="actnbr-action actnbr-actn-following no-display" href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
<span>Subscribed</span>
</a>
<div class="actnbr-popover tip tip-top-left actnbr-notice" id="follow-bubble">
<div class="tip-arrow"></div>
<div class="tip-inner actnbr-follow-bubble">
<ul>
<li class="actnbr-sitename">
<a href="https://blog.acolyer.org">
<img loading='lazy' alt='' src='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png' srcset='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=75&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1.5x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=100&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 2x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=150&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 3x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=200&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 4x' class='avatar avatar-50' height='50' width='50' /> the morning paper </a>
</li>
<div class="actnbr-message no-display"></div>
<form method="post" action="https://subscribe.wordpress.com" accept-charset="utf-8" style="display: none;">
<div class="actnbr-follow-count">Join 3,125 other subscribers</div>
<div>
<input type="email" name="email" placeholder="Enter your email address" class="actnbr-email-field" aria-label="Enter your email address" />
</div>
<input type="hidden" name="action" value="subscribe" />
<input type="hidden" name="blog_id" value="23592848" />
<input type="hidden" name="source" value="https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/" />
<input type="hidden" name="sub-type" value="actionbar-follow" />
<input type="hidden" id="_wpnonce" name="_wpnonce" value="030732bedd" /> <div class="actnbr-button-wrap">
<button type="submit" value="Sign me up">
Sign me up </button>
</div>
</form>
<li class="actnbr-login-nudge">
<div>
Already have a WordPress.com account? <a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fblog.acolyer.org%252F2015%252F10%252F16%252Fholistic-configuration-management-at-facebook%252F">Log in now.</a> </div>
</li>
</ul>
</div>
</div>
</li>
<li class="actnbr-ellipsis actnbr-hidden">
<svg class="gridicon gridicons-ellipsis" height="24" width="24" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M7 12c0 1.104-.896 2-2 2s-2-.896-2-2 .896-2 2-2 2 .896 2 2zm12-2c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2zm-7 0c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2z"/></g></svg> <div class="actnbr-popover tip tip-top-left actnbr-more">
<div class="tip-arrow"></div>
<div class="tip-inner">
<ul>
<li class="actnbr-sitename">
<a href="https://blog.acolyer.org">
<img loading='lazy' alt='' src='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png' srcset='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=75&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1.5x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=100&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 2x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=150&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 3x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=200&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 4x' class='avatar avatar-50' height='50' width='50' /> the morning paper </a>
</li>
<li class="actnbr-folded-follow">
<a class="actnbr-action actnbr-actn-follow " href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
<span>Subscribe</span>
</a>
<a class="actnbr-action actnbr-actn-following no-display" href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
<span>Subscribed</span>
</a>
</li>
<li class="actnbr-signup"><a href="https://wordpress.com/start/">Sign up</a></li>
<li class="actnbr-login"><a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fblog.acolyer.org%252F2015%252F10%252F16%252Fholistic-configuration-management-at-facebook%252F">Log in</a></li>
<li class="actnbr-shortlink">
<a href="https://wp.me/p1AZzO-jV">
<span class="actnbr-shortlink__text">Copy shortlink</span>
<span class="actnbr-shortlink__icon"><svg class="gridicon gridicons-checkmark" height="16" width="16" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M9 19.414l-6.707-6.707 1.414-1.414L9 16.586 20.293 5.293l1.414 1.414"/></g></svg></span>
</a>
</li>
<li class="flb-report">
<a href="https://wordpress.com/abuse/?report_url=https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/" target="_blank" rel="noopener noreferrer">
Report this content </a>
</li>
<li class="actnbr-reader">
<a href="https://wordpress.com/reader/blogs/23592848/posts/1235">
View post in Reader </a>
</li>
<li class="actnbr-subs">
<a href="https://subscribe.wordpress.com/">Manage subscriptions</a>
</li>
<li class="actnbr-fold"><a href="">Collapse this bar</a></li>
</ul>
</div>
</div>
</li>
</ul>
</div>
<script>
window.addEventListener( "DOMContentLoaded", function( event ) {
var link = document.createElement( "link" );
link.href = "/wp-content/mu-plugins/actionbar/actionbar.css?v=20250116";
link.type = "text/css";
link.rel = "stylesheet";
document.head.appendChild( link );
var script = document.createElement( "script" );
script.src = "/wp-content/mu-plugins/actionbar/actionbar.js?v=20250204";
document.body.appendChild( script );
} );
</script>
<div id="jp-carousel-loading-overlay" style="display: none;">
<div id="jp-carousel-loading-wrapper">
<span id="jp-carousel-library-loading"><svg class="jetpack-spinner" width="40" height="40" viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><circle cx="50" cy="50" r="46" fill="none" stroke="#ddd" stroke-width="8"/><path d="M 50 4 A 46 46 0 0 1 96 50" fill="none" stroke="currentColor" stroke-width="8" stroke-linecap="round"><animateTransform attributeName="transform" type="rotate" dur="1.4s" from="0 50 50" to="360 50 50" repeatCount="indefinite"/></path></svg></span>
</div>
</div>
<div class="jp-carousel-overlay" style="display: none;">
<div class="jp-carousel-container">
<!-- The Carousel Swiper -->
<div
class="jp-carousel-wrap swiper jp-carousel-swiper-container jp-carousel-transitions"
itemscope
itemtype="https://schema.org/ImageGallery">
<div class="jp-carousel swiper-wrapper"></div>
<div class="jp-swiper-button-prev swiper-button-prev">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="maskPrev" mask-type="alpha" maskUnits="userSpaceOnUse" x="8" y="6" width="9" height="12">
<path d="M16.2072 16.59L11.6496 12L16.2072 7.41L14.8041 6L8.8335 12L14.8041 18L16.2072 16.59Z" fill="white"/>
</mask>
<g mask="url(#maskPrev)">
<rect x="0.579102" width="23.8823" height="24" fill="#FFFFFF"/>
</g>
</svg>
</div>
<div class="jp-swiper-button-next swiper-button-next">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="maskNext" mask-type="alpha" maskUnits="userSpaceOnUse" x="8" y="6" width="8" height="12">
<path d="M8.59814 16.59L13.1557 12L8.59814 7.41L10.0012 6L15.9718 12L10.0012 18L8.59814 16.59Z" fill="white"/>
</mask>
<g mask="url(#maskNext)">
<rect x="0.34375" width="23.8822" height="24" fill="#FFFFFF"/>
</g>
</svg>
</div>
</div>
<!-- The main close buton -->
<div class="jp-carousel-close-hint">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="maskClose" mask-type="alpha" maskUnits="userSpaceOnUse" x="5" y="5" width="15" height="14">
<path d="M19.3166 6.41L17.9135 5L12.3509 10.59L6.78834 5L5.38525 6.41L10.9478 12L5.38525 17.59L6.78834 19L12.3509 13.41L17.9135 19L19.3166 17.59L13.754 12L19.3166 6.41Z" fill="white"/>
</mask>
<g mask="url(#maskClose)">
<rect x="0.409668" width="23.8823" height="24" fill="#FFFFFF"/>
</g>
</svg>
</div>
<!-- Image info, comments and meta -->
<div class="jp-carousel-info">
<div class="jp-carousel-info-footer">
<div class="jp-carousel-pagination-container">
<div class="jp-swiper-pagination swiper-pagination"></div>
<div class="jp-carousel-pagination"></div>
</div>
<div class="jp-carousel-photo-title-container">
<div class="jp-carousel-photo-caption"></div>
</div>
<div class="jp-carousel-photo-icons-container">
<a href="#" class="jp-carousel-icon-btn jp-carousel-icon-info" aria-label="Toggle photo metadata visibility">
<span class="jp-carousel-icon">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="maskInfo" mask-type="alpha" maskUnits="userSpaceOnUse" x="2" y="2" width="21" height="20">
<path fill-rule="evenodd" clip-rule="evenodd" d="M12.7537 2C7.26076 2 2.80273 6.48 2.80273 12C2.80273 17.52 7.26076 22 12.7537 22C18.2466 22 22.7046 17.52 22.7046 12C22.7046 6.48 18.2466 2 12.7537 2ZM11.7586 7V9H13.7488V7H11.7586ZM11.7586 11V17H13.7488V11H11.7586ZM4.79292 12C4.79292 16.41 8.36531 20 12.7537 20C17.142 20 20.7144 16.41 20.7144 12C20.7144 7.59 17.142 4 12.7537 4C8.36531 4 4.79292 7.59 4.79292 12Z" fill="white"/>
</mask>
<g mask="url(#maskInfo)">
<rect x="0.8125" width="23.8823" height="24" fill="#FFFFFF"/>
</g>
</svg>
</span>
</a>
<a href="#" class="jp-carousel-icon-btn jp-carousel-icon-comments" aria-label="Toggle photo comments visibility">
<span class="jp-carousel-icon">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="maskComments" mask-type="alpha" maskUnits="userSpaceOnUse" x="2" y="2" width="21" height="20">
<path fill-rule="evenodd" clip-rule="evenodd" d="M4.3271 2H20.2486C21.3432 2 22.2388 2.9 22.2388 4V16C22.2388 17.1 21.3432 18 20.2486 18H6.31729L2.33691 22V4C2.33691 2.9 3.2325 2 4.3271 2ZM6.31729 16H20.2486V4H4.3271V18L6.31729 16Z" fill="white"/>
</mask>
<g mask="url(#maskComments)">
<rect x="0.34668" width="23.8823" height="24" fill="#FFFFFF"/>
</g>
</svg>
<span class="jp-carousel-has-comments-indicator" aria-label="This image has comments."></span>
</span>
</a>
</div>
</div>
<div class="jp-carousel-info-extra">
<div class="jp-carousel-info-content-wrapper">
<div class="jp-carousel-photo-title-container">
<div class="jp-carousel-photo-title"></div>
</div>
<div class="jp-carousel-comments-wrapper">
<div id="jp-carousel-comments-loading">
<span>Loading Comments...</span>
</div>
<div class="jp-carousel-comments"></div>
<div id="jp-carousel-comment-form-container">
<span id="jp-carousel-comment-form-spinner"><svg class="jetpack-spinner" width="20" height="20" viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><circle cx="50" cy="50" r="46" fill="none" stroke="#ddd" stroke-width="8"/><path d="M 50 4 A 46 46 0 0 1 96 50" fill="none" stroke="currentColor" stroke-width="8" stroke-linecap="round"><animateTransform attributeName="transform" type="rotate" dur="1.4s" from="0 50 50" to="360 50 50" repeatCount="indefinite"/></path></svg></span>
<div id="jp-carousel-comment-post-results"></div>
<form id="jp-carousel-comment-form">
<label for="jp-carousel-comment-form-comment-field" class="screen-reader-text">Write a Comment...</label>
<textarea
name="comment"
class="jp-carousel-comment-form-field jp-carousel-comment-form-textarea"
id="jp-carousel-comment-form-comment-field"
placeholder="Write a Comment..."
></textarea>
<div id="jp-carousel-comment-form-submit-and-info-wrapper">
<div id="jp-carousel-comment-form-commenting-as">
<fieldset>
<label for="jp-carousel-comment-form-email-field">Email (Required)</label>
<input type="text" name="email" class="jp-carousel-comment-form-field jp-carousel-comment-form-text-field" id="jp-carousel-comment-form-email-field" />
</fieldset>
<fieldset>
<label for="jp-carousel-comment-form-author-field">Name (Required)</label>
<input type="text" name="author" class="jp-carousel-comment-form-field jp-carousel-comment-form-text-field" id="jp-carousel-comment-form-author-field" />
</fieldset>
<fieldset>
<label for="jp-carousel-comment-form-url-field">Website</label>
<input type="text" name="url" class="jp-carousel-comment-form-field jp-carousel-comment-form-text-field" id="jp-carousel-comment-form-url-field" />
</fieldset>
</div>
<input
type="submit"
name="submit"
class="jp-carousel-comment-form-button"
id="jp-carousel-comment-form-button-submit"
value="Post Comment" />
</div>
</form>
</div>
</div>
<div class="jp-carousel-image-meta">
<div class="jp-carousel-title-and-caption">
<div class="jp-carousel-photo-info">
<div class="jp-carousel-caption" itemprop="caption description"></div>
</div>
<div class="jp-carousel-photo-description"></div>
</div>
<a class="jp-carousel-image-download" href="#" aria-label="Download image" target="_blank" style="display: none;">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="mask0" mask-type="alpha" maskUnits="userSpaceOnUse" x="3" y="3" width="19" height="18">
<path fill-rule="evenodd" clip-rule="evenodd" d="M5.84615 5V19H19.7775V12H21.7677V19C21.7677 20.1 20.8721 21 19.7775 21H5.84615C4.74159 21 3.85596 20.1 3.85596 19V5C3.85596 3.9 4.74159 3 5.84615 3H12.8118V5H5.84615ZM14.802 5V3H21.7677V10H19.7775V6.41L9.99569 16.24L8.59261 14.83L18.3744 5H14.802Z" fill="white"/>
</mask>
<g mask="url(#mask0)">
<rect x="0.870605" width="23.8823" height="24" fill="#FFFFFF"/>
</g>
</svg>
<span class="jp-carousel-download-text"></span>
</a>
<div class="jp-carousel-image-map" style="display: none;"></div>
</div>
</div>
</div>
</div>
</div>
</div>
<script type="text/javascript">
window.WPCOM_sharing_counts = {"https://blog.acolyer.org/2015/10/16/holistic-configuration-management-at-facebook/":1235};
</script>
<script type="text/javascript">
(function () {
var wpcom_reblog = {
source: 'toolbar',
toggle_reblog_box_flair: function (obj_id, post_id) {
// Go to site selector. This will redirect to their blog if they only have one.
const postEndpoint = `https://wordpress.com/post`;
// Ideally we would use the permalink here, but fortunately this will be replaced with the
// post permalink in the editor.
const originalURL = `${ document.location.href }?page_id=${ post_id }`;
const url =
postEndpoint +
'?url=' +
encodeURIComponent( originalURL ) +
'&is_post_share=true' +
'&v=5';
const redirect = function () {
if (
! window.open( url, '_blank' )
) {
location.href = url;
}
};
if ( /Firefox/.test( navigator.userAgent ) ) {
setTimeout( redirect, 0 );
} else {
redirect();
}
},
};
window.wpcom_reblog = wpcom_reblog;
})();
</script>
<script id="jetpack-carousel-js-extra">
var jetpackSwiperLibraryPath = {"url":"/wp-content/mu-plugins/jetpack-plugin/moon/_inc/blocks/swiper.js"};
var jetpackCarouselStrings = {"widths":[370,700,1000,1200,1400,2000],"is_logged_in":"","lang":"en","ajaxurl":"https://blog.acolyer.org/wp-admin/admin-ajax.php","nonce":"1c8af60010","display_exif":"1","display_comments":"1","single_image_gallery":"1","single_image_gallery_media_file":"","background_color":"black","comment":"Comment","post_comment":"Post Comment","write_comment":"Write a Comment...","loading_comments":"Loading Comments...","image_label":"Open image in full-screen.","download_original":"View full size \u003Cspan class=\"photo-size\"\u003E{0}\u003Cspan class=\"photo-size-times\"\u003E\u00d7\u003C/span\u003E{1}\u003C/span\u003E","no_comment_text":"Please be sure to submit some text with your comment.","no_comment_email":"Please provide an email address to comment.","no_comment_author":"Please provide your name to comment.","comment_post_error":"Sorry, but there was an error posting your comment. Please try again later.","comment_approved":"Your comment was approved.","comment_unapproved":"Your comment is in moderation.","camera":"Camera","aperture":"Aperture","shutter_speed":"Shutter Speed","focal_length":"Focal Length","copyright":"Copyright","comment_registration":"0","require_name_email":"1","login_url":"https://adriancolyer.wordpress.com/wp-login.php?redirect_to=https%3A%2F%2Fblog.acolyer.org%2F2015%2F10%2F16%2Fholistic-configuration-management-at-facebook%2F","blog_id":"23592848","meta_data":["camera","aperture","shutter_speed","focal_length","copyright"],"stats_query_args":"blog=23592848&v=wpcom&tz=1&user_id=0&subd=adriancolyer","is_public":"1"};
//# sourceURL=jetpack-carousel-js-extra
</script>
<script id="sharing-js-js-extra">
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":"1"};
//# sourceURL=sharing-js-js-extra
</script>
<script crossorigin='anonymous' type='text/javascript' src='/_static/??-eJydj0sOwjAMRC9EMCDRlgXiKCgkbus2cUKd8Lk9CLWoghXsPNab0QxcozKBE3ICn1V0uSEWcNSjwDljxlazdTgsO1nADE4t+icS8wkisphBJ9OqDXQCrC/U6ESBfzDN9Q826SkqR9yrOpgsqqbbp3s2q8MUtelHDT4EhiOxgVMmZ8HoIWRB9+amx9LT15ZZ7ISpazTBf8j/60irB7Ta2vvrJG7GHge/X5dVuduuqqLoHmPhp/8='></script>
<script id="sharing-js-js-after">
window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
window.WPCOM_sharing_popups[ "twitter" ] = "menubar=1,resizable=1,width=600,height=350";
window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
window.WPCOM_sharing_popups[ "linkedin" ] = "menubar=1,resizable=1,width=580,height=450";
//# sourceURL=sharing-js-js-after
</script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/72x72/","ext":".png","svgUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/svg/","svgExt":".svg","source":{"concatemoji":"/wp-includes/js/wp-emoji-release.min.js?m=1783090810i&ver=7.1.1-alpha-63555"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=/wp-includes/js/wp-emoji-loader.min.js
</script>
<iframe src='https://widgets.wp.com/likes/master.html?ver=20260911#ver=20260911' scrolling='no' id='likes-master' name='likes-master' style='display:none;'></iframe>
<div id='likes-other-gravatars' role="dialog" aria-hidden="true" tabindex="-1"><div class="likes-text"><span>%d</span></div><ul class="wpl-avatars sd-like-gravatars"></ul></div>
<script src="//stats.wp.com/w.js?69" defer></script> <script type="text/javascript">
_tkq = window._tkq || [];
_stq = window._stq || [];
_tkq.push(['storeContext', {'blog_id':'23592848','blog_tz':'1','user_lang':'en','blog_lang':'en','user_id':'0'}]);
// Prevent sending pageview tracking from WP-Admin pages.
_stq.push(['view', {'blog':'23592848','v':'wpcom','tz':'1','user_id':'0','post':'1235','subd':'adriancolyer'}]);
_stq.push(['extra', {'crypt':'UE5tW3cvZGRDbzJxMWx6aVlydldVeXIvdUpWanJmOVg4Mys2XWxoN19+Lk5YJUp6QyVxQW02Q0lDM1lFMHBvZkEwd0d2SF9scEhaUWomeXxRUl8zNWRkMX5zZXFOYko4Y2kuUnRtODRdQT1jOS09cGgwSl0/RzkvP3xicmtGWk1EVFV3R3YlQ3loMFVoLzFTan5DdEF5JUw5TlRzLHpURWYyNTdDNVpEcnRtTERkV3dpLyxBcS5wTm5Vbm1GcjImN1c9UmVDUj1kTnZCLHVBfH56TzM1LmdQeFUyUlB1Sg=='}]);
_stq.push([ 'clickTrackerInit', '23592848', '1235' ]);
</script>
<noscript><img src="https://pixel.wp.com/b.gif?v=noscript" style="height:1px;width:1px;overflow:hidden;position:absolute;bottom:1px;" alt="" /></noscript>
<meta id="bilmur" property="bilmur:data" content="" data-provider="wordpress.com" data-service="simple" data-site-tz="Europe/London" data-custom-props="{&quot;logged_in&quot;:&quot;0&quot;,&quot;wptheme&quot;:&quot;pub\/penscratch-2&quot;,&quot;wptheme_is_block&quot;:&quot;0&quot;}" >
<script defer src="/wp-content/js/bilmur.min.js?i=17&amp;m=202637"></script>
</body>
</html>