805 lines
88 KiB
HTML
805 lines
88 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<link rel="profile" href="http://gmpg.org/xfn/11">
|
||
|
||
<title>Maelstrom: mitigating datacenter-level disasters by draining interdependent traffic safely and efficiently – the morning paper</title>
|
||
<meta name='robots' content='max-image-preview:large' />
|
||
|
||
<!-- Async WordPress.com Remote Login -->
|
||
<script id="wpcom_remote_login_js">
|
||
var wpcom_remote_login_extra_auth = '';
|
||
function wpcom_remote_login_remove_dom_node_id( element_id ) {
|
||
var dom_node = document.getElementById( element_id );
|
||
if ( dom_node ) { dom_node.parentNode.removeChild( dom_node ); }
|
||
}
|
||
function wpcom_remote_login_remove_dom_node_classes( class_name ) {
|
||
var dom_nodes = document.querySelectorAll( '.' + class_name );
|
||
for ( var i = 0; i < dom_nodes.length; i++ ) {
|
||
dom_nodes[ i ].parentNode.removeChild( dom_nodes[ i ] );
|
||
}
|
||
}
|
||
function wpcom_remote_login_final_cleanup() {
|
||
wpcom_remote_login_remove_dom_node_classes( "wpcom_remote_login_msg" );
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_validate" );
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_js" );
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_iframe" );
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_styles" );
|
||
}
|
||
|
||
// Watch for messages back from the remote login
|
||
window.addEventListener( "message", function( e ) {
|
||
if ( e.origin === "https://r-login.wordpress.com" ) {
|
||
var data = {};
|
||
try {
|
||
data = JSON.parse( e.data );
|
||
} catch( e ) {
|
||
wpcom_remote_login_final_cleanup();
|
||
return;
|
||
}
|
||
|
||
if ( data.msg === 'LOGIN' ) {
|
||
// Clean up the login check iframe
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
|
||
|
||
var id_regex = new RegExp( /^[0-9]+$/ );
|
||
var token_regex = new RegExp( /^.*|.*|.*$/ );
|
||
if (
|
||
token_regex.test( data.token )
|
||
&& id_regex.test( data.wpcomid )
|
||
) {
|
||
// We have everything we need to ask for a login
|
||
var script = document.createElement( "script" );
|
||
script.setAttribute( "id", "wpcom_remote_login_validate" );
|
||
script.src = '/remote-login.php?wpcom_remote_login=validate'
|
||
+ '&wpcomid=' + data.wpcomid
|
||
+ '&token=' + encodeURIComponent( data.token )
|
||
+ '&host=' + window.location.protocol
|
||
+ '//' + window.location.hostname
|
||
+ '&postid=7797'
|
||
+ '&is_singular=1';
|
||
document.body.appendChild( script );
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
// Safari ITP, not logged in, so redirect
|
||
if ( data.msg === 'LOGIN-REDIRECT' ) {
|
||
window.location = 'https://wordpress.com/log-in?redirect_to=' + window.location.href;
|
||
return;
|
||
}
|
||
|
||
// Safari ITP, storage access failed, remove the request
|
||
if ( data.msg === 'LOGIN-REMOVE' ) {
|
||
var css_zap = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } @media screen and ( max-width: 782px ) { html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } }';
|
||
var style_zap = document.createElement( 'style' );
|
||
style_zap.type = 'text/css';
|
||
style_zap.appendChild( document.createTextNode( css_zap ) );
|
||
document.body.appendChild( style_zap );
|
||
|
||
var e = document.getElementById( 'wpcom_request_access_iframe' );
|
||
e.parentNode.removeChild( e );
|
||
|
||
document.cookie = 'wordpress_com_login_access=denied; path=/; max-age=31536000';
|
||
|
||
return;
|
||
}
|
||
|
||
// Safari ITP
|
||
if ( data.msg === 'REQUEST_ACCESS' ) {
|
||
console.log( 'request access: safari' );
|
||
|
||
// Check ITP iframe enable/disable knob
|
||
if ( wpcom_remote_login_extra_auth !== 'safari_itp_iframe' ) {
|
||
return;
|
||
}
|
||
|
||
// If we are in a "private window" there is no ITP.
|
||
var private_window = false;
|
||
try {
|
||
var opendb = window.openDatabase( null, null, null, null );
|
||
} catch( e ) {
|
||
private_window = true;
|
||
}
|
||
|
||
if ( private_window ) {
|
||
console.log( 'private window' );
|
||
return;
|
||
}
|
||
|
||
var iframe = document.createElement( 'iframe' );
|
||
iframe.id = 'wpcom_request_access_iframe';
|
||
iframe.setAttribute( 'scrolling', 'no' );
|
||
iframe.setAttribute( 'sandbox', 'allow-storage-access-by-user-activation allow-scripts allow-same-origin allow-top-navigation-by-user-activation' );
|
||
iframe.src = 'https://r-login.wordpress.com/remote-login.php?wpcom_remote_login=request_access&origin=' + encodeURIComponent( data.origin ) + '&wpcomid=' + encodeURIComponent( data.wpcomid );
|
||
|
||
var css = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 46px !important; } * html body { margin-top: 46px !important; } @media screen and ( max-width: 660px ) { html { margin-top: 71px !important; } * html body { margin-top: 71px !important; } #wpcom_request_access_iframe { display: block; height: 71px !important; } } #wpcom_request_access_iframe { border: 0px; height: 46px; position: fixed; top: 0; left: 0; width: 100%; min-width: 100%; z-index: 99999; background: #23282d; } ';
|
||
|
||
var style = document.createElement( 'style' );
|
||
style.type = 'text/css';
|
||
style.id = 'wpcom_request_access_styles';
|
||
style.appendChild( document.createTextNode( css ) );
|
||
document.body.appendChild( style );
|
||
|
||
document.body.appendChild( iframe );
|
||
}
|
||
|
||
if ( data.msg === 'DONE' ) {
|
||
wpcom_remote_login_final_cleanup();
|
||
}
|
||
}
|
||
}, false );
|
||
|
||
// Inject the remote login iframe after the page has had a chance to load
|
||
// more critical resources
|
||
window.addEventListener( "DOMContentLoaded", function( e ) {
|
||
var iframe = document.createElement( "iframe" );
|
||
iframe.style.display = "none";
|
||
iframe.setAttribute( "scrolling", "no" );
|
||
iframe.setAttribute( "id", "wpcom_remote_login_key" );
|
||
iframe.src = "https://r-login.wordpress.com/remote-login.php"
|
||
+ "?wpcom_remote_login=key"
|
||
+ "&origin=aHR0cHM6Ly9ibG9nLmFjb2x5ZXIub3Jn"
|
||
+ "&wpcomid=23592848"
|
||
+ "&time=" + Math.floor( Date.now() / 1000 );
|
||
document.body.appendChild( iframe );
|
||
}, false );
|
||
</script>
|
||
<link rel='dns-prefetch' href='//s2.wp.com' />
|
||
<link rel='dns-prefetch' href='//fonts-api.wp.com' />
|
||
<link rel='dns-prefetch' href='//s1.wp.com' />
|
||
<link rel='dns-prefetch' href='//s0.wp.com' />
|
||
<link rel="alternate" type="application/rss+xml" title="the morning paper » Feed" href="https://blog.acolyer.org/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="the morning paper » Comments Feed" href="https://blog.acolyer.org/comments/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="the morning paper » Maelstrom: mitigating datacenter-level disasters by draining interdependent traffic safely and efficiently Comments Feed" href="https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/feed/" />
|
||
<script type="text/javascript">
|
||
/* <![CDATA[ */
|
||
function addLoadEvent(func) {
|
||
var oldonload = window.onload;
|
||
if (typeof window.onload != 'function') {
|
||
window.onload = func;
|
||
} else {
|
||
window.onload = function () {
|
||
oldonload();
|
||
func();
|
||
}
|
||
}
|
||
}
|
||
/* ]]> */
|
||
</script>
|
||
<style id="wp-img-auto-sizes-contain-inline-css">
|
||
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
|
||
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
|
||
</style>
|
||
<link crossorigin='anonymous' rel='stylesheet' id='all-css-2-1' href='/_static/??-eJxtjEsKgDAMBS9kjbqwuBDPUtog9o9J8fpaEQSRBw8GhoEjC50iY2QIRWRf1i0SWOSstHsYQkr1TPFIsKNXjEbkRPyhVhM18J/0m8M3fFPVlzD3UnbTtWG0J2CTNNk=&cssminify=yes' type='text/css' media='all' />
|
||
<style id="wp-emoji-styles-inline-css">
|
||
|
||
img.wp-smiley, img.emoji {
|
||
display: inline !important;
|
||
border: none !important;
|
||
box-shadow: none !important;
|
||
height: 1em !important;
|
||
width: 1em !important;
|
||
margin: 0 0.07em !important;
|
||
vertical-align: -0.1em !important;
|
||
background: none !important;
|
||
padding: 0 !important;
|
||
}
|
||
/*# sourceURL=wp-emoji-styles-inline-css */
|
||
</style>
|
||
<style id="wp-block-library-inline-css">
|
||
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
|
||
|
||
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/common.min.css */
|
||
</style>
|
||
<style id="wp-block-library-inline-css-extra">
|
||
.has-text-align-justify {
|
||
text-align:justify;
|
||
}
|
||
.has-text-align-justify{text-align:justify;}
|
||
/*# sourceURL=wp-block-library-inline-css */
|
||
</style>
|
||
|
||
<style id="classic-theme-styles-inline-css">
|
||
.wp-block-button__link{background-color:#32373c;border-radius:9999px;box-shadow:none;color:#fff;font-size:1.125em;padding:calc(.667em + 2px) calc(1.333em + 2px);text-decoration:none}.wp-block-file__button{background:#32373c;color:#fff}.wp-block-accordion-heading{margin:0}.wp-block-accordion-heading__toggle{background-color:inherit!important;color:inherit!important}.wp-block-accordion-heading__toggle:not(:focus-visible){outline:none}.wp-block-accordion-heading__toggle:focus,.wp-block-accordion-heading__toggle:hover{background-color:inherit!important;border:none;box-shadow:none;color:inherit;padding:var(--wp--preset--spacing--20,1em) 0;text-decoration:none}.wp-block-accordion-heading__toggle:focus-visible{outline:auto;outline-offset:0}.wp-block-tab:not(.has-text-color){color:inherit!important}.wp-block-tab:not(.has-background){background-color:inherit!important}.wp-block-tab:focus,.wp-block-tab:hover{text-decoration:none}.wp-block-tab:focus-visible{outline:auto;outline-offset:0}
|
||
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/classic.min.css */
|
||
</style>
|
||
|
||
<style id="coblocks-inline-format-inline-css">
|
||
.uppercase{text-transform:uppercase}
|
||
/*# sourceURL=coblocks-inline-format-inline-css */
|
||
</style>
|
||
<link crossorigin='anonymous' rel='stylesheet' id='all-css-12-1' href='/_static/??/wp-content/mu-plugins/core-compat/wp-mediaelement.css,/wp-content/mu-plugins/wpcom-bbpress-premium-themes.css?m=1432920480j&cssminify=yes' type='text/css' media='all' />
|
||
<style id="global-styles-inline-css">
|
||
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #fff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--color--dark-green: #1c7c7c;--wp--preset--color--dark-gray: #666;--wp--preset--color--medium-gray: #999;--wp--preset--color--light-gray: #eee;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--font-family--albert-sans: 'Albert Sans', sans-serif;--wp--preset--font-family--alegreya: Alegreya, serif;--wp--preset--font-family--arvo: Arvo, serif;--wp--preset--font-family--bodoni-moda: 'Bodoni Moda', serif;--wp--preset--font-family--bricolage-grotesque: 'Bricolage Grotesque', sans-serif;--wp--preset--font-family--cabin: Cabin, sans-serif;--wp--preset--font-family--chivo: Chivo, sans-serif;--wp--preset--font-family--commissioner: Commissioner, sans-serif;--wp--preset--font-family--cormorant: Cormorant, serif;--wp--preset--font-family--courier-prime: 'Courier Prime', monospace;--wp--preset--font-family--crimson-pro: 'Crimson Pro', serif;--wp--preset--font-family--dm-mono: 'DM Mono', monospace;--wp--preset--font-family--dm-sans: 'DM Sans', sans-serif;--wp--preset--font-family--dm-serif-display: 'DM Serif Display', serif;--wp--preset--font-family--domine: Domine, serif;--wp--preset--font-family--eb-garamond: 'EB Garamond', serif;--wp--preset--font-family--epilogue: Epilogue, sans-serif;--wp--preset--font-family--fahkwang: Fahkwang, sans-serif;--wp--preset--font-family--figtree: Figtree, sans-serif;--wp--preset--font-family--fira-sans: 'Fira Sans', sans-serif;--wp--preset--font-family--fjalla-one: 'Fjalla One', sans-serif;--wp--preset--font-family--fraunces: Fraunces, serif;--wp--preset--font-family--gabarito: Gabarito, system-ui;--wp--preset--font-family--ibm-plex-mono: 'IBM Plex Mono', monospace;--wp--preset--font-family--ibm-plex-sans: 'IBM Plex Sans', sans-serif;--wp--preset--font-family--ibarra-real-nova: 'Ibarra Real Nova', serif;--wp--preset--font-family--instrument-serif: 'Instrument Serif', serif;--wp--preset--font-family--inter: Inter, sans-serif;--wp--preset--font-family--josefin-sans: 'Josefin Sans', sans-serif;--wp--preset--font-family--jost: Jost, sans-serif;--wp--preset--font-family--libre-baskerville: 'Libre Baskerville', serif;--wp--preset--font-family--libre-franklin: 'Libre Franklin', sans-serif;--wp--preset--font-family--literata: Literata, serif;--wp--preset--font-family--lora: Lora, serif;--wp--preset--font-family--merriweather: Merriweather, serif;--wp--preset--font-family--montserrat: Montserrat, sans-serif;--wp--preset--font-family--newsreader: Newsreader, serif;--wp--preset--font-family--noto-sans-mono: 'Noto Sans Mono', sans-serif;--wp--preset--font-family--nunito: Nunito, sans-serif;--wp--preset--font-family--open-sans: 'Open Sans', sans-serif;--wp--preset--font-family--overpass: Overpass, sans-serif;--wp--preset--font-family--pt-serif: 'PT Serif', serif;--wp--preset--font-family--petrona: Petrona, serif;--wp--preset--font-family--piazzolla: Piazzolla, serif;--wp--preset--font-family--playfair-display: 'Playfair Display', serif;--wp--preset--font-family--plus-jakarta-sans: 'Plus Jakarta Sans', sans-serif;--wp--preset--font-family--poppins: Poppins, sans-serif;--wp--preset--font-family--raleway: Raleway, sans-serif;--wp--preset--font-family--roboto: Roboto, sans-serif;--wp--preset--font-family--roboto-slab: 'Roboto Slab', serif;--wp--preset--font-family--rubik: Rubik, sans-serif;--wp--preset--font-family--rufina: Rufina, serif;--wp--preset--font-family--sora: Sora, sans-serif;--wp--preset--font-family--source-sans-3: 'Source Sans 3', sans-serif;--wp--preset--font-family--source-serif-4: 'Source Serif 4', serif;--wp--preset--font-family--space-mono: 'Space Mono', monospace;--wp--preset--font-family--syne: Syne, sans-serif;--wp--preset--font-family--texturina: Texturina, serif;--wp--preset--font-family--urbanist: Urbanist, sans-serif;--wp--preset--font-family--work-sans: 'Work Sans', sans-serif;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-color{color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-color{color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-color{color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-color{color: var(--wp--preset--color--light-gray) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-background-color{background-color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-background-color{background-color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-background-color{background-color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-background-color{background-color: var(--wp--preset--color--light-gray) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-border-color{border-color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-border-color{border-color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-border-color{border-color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-border-color{border-color: var(--wp--preset--color--light-gray) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}.has-albert-sans-font-family{font-family: var(--wp--preset--font-family--albert-sans) !important;}.has-alegreya-font-family{font-family: var(--wp--preset--font-family--alegreya) !important;}.has-arvo-font-family{font-family: var(--wp--preset--font-family--arvo) !important;}.has-bodoni-moda-font-family{font-family: var(--wp--preset--font-family--bodoni-moda) !important;}.has-bricolage-grotesque-font-family{font-family: var(--wp--preset--font-family--bricolage-grotesque) !important;}.has-cabin-font-family{font-family: var(--wp--preset--font-family--cabin) !important;}.has-chivo-font-family{font-family: var(--wp--preset--font-family--chivo) !important;}.has-commissioner-font-family{font-family: var(--wp--preset--font-family--commissioner) !important;}.has-cormorant-font-family{font-family: var(--wp--preset--font-family--cormorant) !important;}.has-courier-prime-font-family{font-family: var(--wp--preset--font-family--courier-prime) !important;}.has-crimson-pro-font-family{font-family: var(--wp--preset--font-family--crimson-pro) !important;}.has-dm-mono-font-family{font-family: var(--wp--preset--font-family--dm-mono) !important;}.has-dm-sans-font-family{font-family: var(--wp--preset--font-family--dm-sans) !important;}.has-dm-serif-display-font-family{font-family: var(--wp--preset--font-family--dm-serif-display) !important;}.has-domine-font-family{font-family: var(--wp--preset--font-family--domine) !important;}.has-eb-garamond-font-family{font-family: var(--wp--preset--font-family--eb-garamond) !important;}.has-epilogue-font-family{font-family: var(--wp--preset--font-family--epilogue) !important;}.has-fahkwang-font-family{font-family: var(--wp--preset--font-family--fahkwang) !important;}.has-figtree-font-family{font-family: var(--wp--preset--font-family--figtree) !important;}.has-fira-sans-font-family{font-family: var(--wp--preset--font-family--fira-sans) !important;}.has-fjalla-one-font-family{font-family: var(--wp--preset--font-family--fjalla-one) !important;}.has-fraunces-font-family{font-family: var(--wp--preset--font-family--fraunces) !important;}.has-gabarito-font-family{font-family: var(--wp--preset--font-family--gabarito) !important;}.has-ibm-plex-mono-font-family{font-family: var(--wp--preset--font-family--ibm-plex-mono) !important;}.has-ibm-plex-sans-font-family{font-family: var(--wp--preset--font-family--ibm-plex-sans) !important;}.has-ibarra-real-nova-font-family{font-family: var(--wp--preset--font-family--ibarra-real-nova) !important;}.has-instrument-serif-font-family{font-family: var(--wp--preset--font-family--instrument-serif) !important;}.has-inter-font-family{font-family: var(--wp--preset--font-family--inter) !important;}.has-josefin-sans-font-family{font-family: var(--wp--preset--font-family--josefin-sans) !important;}.has-jost-font-family{font-family: var(--wp--preset--font-family--jost) !important;}.has-libre-baskerville-font-family{font-family: var(--wp--preset--font-family--libre-baskerville) !important;}.has-libre-franklin-font-family{font-family: var(--wp--preset--font-family--libre-franklin) !important;}.has-literata-font-family{font-family: var(--wp--preset--font-family--literata) !important;}.has-lora-font-family{font-family: var(--wp--preset--font-family--lora) !important;}.has-merriweather-font-family{font-family: var(--wp--preset--font-family--merriweather) !important;}.has-montserrat-font-family{font-family: var(--wp--preset--font-family--montserrat) !important;}.has-newsreader-font-family{font-family: var(--wp--preset--font-family--newsreader) !important;}.has-noto-sans-mono-font-family{font-family: var(--wp--preset--font-family--noto-sans-mono) !important;}.has-nunito-font-family{font-family: var(--wp--preset--font-family--nunito) !important;}.has-open-sans-font-family{font-family: var(--wp--preset--font-family--open-sans) !important;}.has-overpass-font-family{font-family: var(--wp--preset--font-family--overpass) !important;}.has-pt-serif-font-family{font-family: var(--wp--preset--font-family--pt-serif) !important;}.has-petrona-font-family{font-family: var(--wp--preset--font-family--petrona) !important;}.has-piazzolla-font-family{font-family: var(--wp--preset--font-family--piazzolla) !important;}.has-playfair-display-font-family{font-family: var(--wp--preset--font-family--playfair-display) !important;}.has-plus-jakarta-sans-font-family{font-family: var(--wp--preset--font-family--plus-jakarta-sans) !important;}.has-poppins-font-family{font-family: var(--wp--preset--font-family--poppins) !important;}.has-raleway-font-family{font-family: var(--wp--preset--font-family--raleway) !important;}.has-roboto-font-family{font-family: var(--wp--preset--font-family--roboto) !important;}.has-roboto-slab-font-family{font-family: var(--wp--preset--font-family--roboto-slab) !important;}.has-rubik-font-family{font-family: var(--wp--preset--font-family--rubik) !important;}.has-rufina-font-family{font-family: var(--wp--preset--font-family--rufina) !important;}.has-sora-font-family{font-family: var(--wp--preset--font-family--sora) !important;}.has-source-sans-3-font-family{font-family: var(--wp--preset--font-family--source-sans-3) !important;}.has-source-serif-4-font-family{font-family: var(--wp--preset--font-family--source-serif-4) !important;}.has-space-mono-font-family{font-family: var(--wp--preset--font-family--space-mono) !important;}.has-syne-font-family{font-family: var(--wp--preset--font-family--syne) !important;}.has-texturina-font-family{font-family: var(--wp--preset--font-family--texturina) !important;}.has-urbanist-font-family{font-family: var(--wp--preset--font-family--urbanist) !important;}.has-work-sans-font-family{font-family: var(--wp--preset--font-family--work-sans) !important;}
|
||
/*# sourceURL=global-styles-inline-css */
|
||
</style>
|
||
|
||
<link crossorigin='anonymous' rel='stylesheet' id='all-css-14-1' href='/_static/??/wp-content/themes/pub/penscratch-2/css/reset.css,/wp-content/themes/pub/penscratch-2/style.css?m=1779214100j&cssminify=yes' type='text/css' media='all' />
|
||
<style id="penscratch-2-style-inline-css">
|
||
.cat-links, .byline .author, .group-blog .byline + .sep, .single .byline + .sep { clip: rect(1px, 1px, 1px, 1px); height: 1px; position: absolute; overflow: hidden; width: 1px; }
|
||
/*# sourceURL=penscratch-2-style-inline-css */
|
||
</style>
|
||
<link rel='stylesheet' id='penscratch-2-fonts-css' href='https://fonts-api.wp.com/css?family=Roboto+Slab%3A300%2C400%2C700&subset=latin%2Clatin-ext' media='all' />
|
||
<link crossorigin='anonymous' rel='stylesheet' id='all-css-16-1' href='/_static/??-eJyNjsEKAjEMRH/ImO0iLB7Eb+mG2K22aWmyLP691YuIIN7mHd7M4FaBihiLoS2cWbGuM1YWpeaNFhiRVHFOhW6673GHfyhRCNXuiWGrVPKXl1eoaQ1RFBv37tBjeA298ZcUuEB/5C0W+QC4JB/bUz3nk5um4+gObhiuD4S6VWY=&cssminify=yes' type='text/css' media='all' />
|
||
<link crossorigin='anonymous' rel='stylesheet' id='print-css-17-1' href='/wp-content/mu-plugins/global-print/global-print.css?m=1465851035i&cssminify=yes' type='text/css' media='print' />
|
||
<style id="jetpack-global-styles-frontend-style-inline-css">
|
||
:root { --font-headings: unset; --font-base: unset; --font-headings-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif; --font-base-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;}
|
||
/*# sourceURL=jetpack-global-styles-frontend-style-inline-css */
|
||
</style>
|
||
<link crossorigin='anonymous' rel='stylesheet' id='all-css-20-1' href='/_static/??-eJyNjcEKAjEMRH/ImhVlqwfxU6TbhrZrmhTTIv69u+JFvHgZ5sHwBh7VeOGG3KB0U6nHzAoztur87cNQRNYInVBBk7tjcCE83zVz3HrVDfxvumb2MPVMAVR8dmRIougX/DhbwrK8pwNEksnROriU884e9+PJDnaYXxwFS20=&cssminify=yes' type='text/css' media='all' />
|
||
<script id="jetpack_related-posts-js-extra">
|
||
var related_posts_js_options = {"post_heading":"h4"};
|
||
//# sourceURL=jetpack_related-posts-js-extra
|
||
</script>
|
||
<script id="wpcom-actionbar-placeholder-js-extra">
|
||
var actionbardata = {"siteID":"23592848","postID":"7797","siteURL":"https://blog.acolyer.org","xhrURL":"https://blog.acolyer.org/wp-admin/admin-ajax.php","nonce":"ec790efa96","isLoggedIn":"","statusMessage":"","subsEmailDefault":"instantly","proxyScriptUrl":"https://s0.wp.com/wp-content/js/wpcom-proxy-request.js?m=1513050504i&ver=20211021","shortlink":"https://wp.me/p1AZzO-21L","i18n":{"followedText":"New posts from this site will now appear in your \u003Ca href=\"https://wordpress.com/reader\"\u003EReader\u003C/a\u003E","foldBar":"Collapse this bar","unfoldBar":"Expand this bar","shortLinkCopied":"Shortlink copied to clipboard."}};
|
||
//# sourceURL=wpcom-actionbar-placeholder-js-extra
|
||
</script>
|
||
<script id="jetpack-mu-wpcom-settings-js-before">
|
||
var JETPACK_MU_WPCOM_SETTINGS = {"assetsUrl":"https://s1.wp.com/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/moon/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/"};
|
||
//# sourceURL=jetpack-mu-wpcom-settings-js-before
|
||
</script>
|
||
<script crossorigin='anonymous' type='text/javascript' src='/_static/??/wp-content/mu-plugins/jetpack-plugin/moon/_inc/build/related-posts/related-posts.min.js,/wp-content/js/rlt-proxy.js?m=1788798821j'></script>
|
||
<script id="rlt-proxy-js-after">
|
||
rltInitialize( {"token":null,"iframeOrigins":["https:\/\/widgets.wp.com"]} );
|
||
//# sourceURL=rlt-proxy-js-after
|
||
</script>
|
||
<link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://adriancolyer.wordpress.com/xmlrpc.php?rsd" />
|
||
<meta name="generator" content="WordPress.com" />
|
||
<link rel="canonical" href="https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/" />
|
||
<link rel='shortlink' href='https://wp.me/p1AZzO-21L' />
|
||
<link rel="alternate" type="application/json+oembed" href="https://public-api.wordpress.com/oembed/?format=json&url=https%3A%2F%2Fblog.acolyer.org%2F2018%2F10%2F24%2Fmaelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently%2F&for=wpcom-auto-discovery" /><link rel="alternate" type="application/xml+oembed" href="https://public-api.wordpress.com/oembed/?format=xml&url=https%3A%2F%2Fblog.acolyer.org%2F2018%2F10%2F24%2Fmaelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently%2F&for=wpcom-auto-discovery" />
|
||
<!-- Jetpack Open Graph Tags -->
|
||
<meta property="og:type" content="article" />
|
||
<meta property="og:title" content="Maelstrom: mitigating datacenter-level disasters by draining interdependent traffic safely and efficiently" />
|
||
<meta property="og:url" content="https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/" />
|
||
<meta property="og:description" content="Maelstrom: mitigating datacenter-level disasters by draining interdependent traffic safely and efficiently Veeraraghavan et al., OSDI’18 Here’s a really valuable paper detailing four plus yea…" />
|
||
<meta property="article:published_time" content="2018-10-24T05:00:00+00:00" />
|
||
<meta property="article:modified_time" content="2018-10-20T06:15:40+00:00" />
|
||
<meta property="og:site_name" content="the morning paper" />
|
||
<meta property="og:image" content="https://blog.acolyer.org/wp-content/uploads/2018/10/maelstrom-table-1.jpeg?w=480" />
|
||
<meta property="og:image:alt" content="" />
|
||
<meta property="og:locale" content="en_US" />
|
||
<meta property="article:publisher" content="https://www.facebook.com/WordPresscom" />
|
||
<meta name="twitter:text:title" content="Maelstrom: mitigating datacenter-level disasters by draining interdependent traffic safely and efficiently" />
|
||
<meta name="twitter:image" content="https://blog.acolyer.org/wp-content/uploads/2018/10/maelstrom-table-1.jpeg?w=144" />
|
||
<meta name="twitter:card" content="summary" />
|
||
|
||
<!-- End Jetpack Open Graph Tags -->
|
||
<link rel="shortcut icon" type="image/x-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=32" sizes="16x16" />
|
||
<link rel="icon" type="image/x-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=32" sizes="16x16" />
|
||
<link rel="apple-touch-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=114" />
|
||
<link rel='openid.server' href='https://blog.acolyer.org/?openidserver=1' />
|
||
<link rel='openid.delegate' href='https://blog.acolyer.org/' />
|
||
<link rel="search" type="application/opensearchdescription+xml" href="https://blog.acolyer.org/osd.xml" title="the morning paper" />
|
||
<link rel="search" type="application/opensearchdescription+xml" href="https://s1.wp.com/opensearch.xml" title="WordPress.com" />
|
||
<meta name="theme-color" content="#ffffff" />
|
||
<meta name="description" content="Maelstrom: mitigating datacenter-level disasters by draining interdependent traffic safely and efficiently Veeraraghavan et al., OSDI'18 Here’s a really valuable paper detailing four plus years of experience dealing with datacenter outages at Facebook. Maelstrom is the system Facebook use in production to mitigate and recover from datacenter-level disasters. The high level idea is simple: drain traffic…" />
|
||
<style id="custom-background-css">
|
||
body.custom-background { background-color: #ffffff; }
|
||
</style>
|
||
<style type="text/css" id="custom-colors-css"> .widget a:hover,
|
||
.site-footer .widget a:hover {
|
||
color: #666;
|
||
}
|
||
.jetpack-social-navigation ul a:hover { color: #FFFFFF;}
|
||
body { background-color: #ffffff;}
|
||
.jetpack-social-navigation ul a:hover { background-color: #1c7c7c;}
|
||
.jetpack-recipe-title { border-color: #1c7c7c;}
|
||
.toggled .menu-toggle, .wp_widget_tag_cloud a:hover, .tags-links a:hover { border-color: #1B7878;}
|
||
.button:hover, .button:focus, .button:active, button:hover, button:focus, button:active, input[type="button"]:hover, input[type="button"]:focus, input[type="button"]:active, input[type="reset"]:hover, input[type="reset"]:focus, input[type="reset"]:active, input[type="submit"]:hover, input[type="submit"]:focus, input[type="submit"]:active, #infinite-handle span:hover { border-color: #1B7878;}
|
||
.button:hover, .button:focus, .button:active, button:hover, button:focus, button:active, input[type="button"]:hover, input[type="button"]:focus, input[type="button"]:active, input[type="reset"]:hover, input[type="reset"]:focus, input[type="reset"]:active, input[type="submit"]:hover, input[type="submit"]:focus, input[type="submit"]:active, #infinite-handle span:hover { color: #1B7878;}
|
||
.menu-toggle:hover #menu-icon, .toggled #menu-icon { fill: #1B7878;}
|
||
a, a:visited, .site-footer a:hover, .main-navigation li:hover > a, .current_page_item a, .current-menu-item a, .toggled .menu-toggle, .entry-footer a:hover, .entry-meta a:hover, .entry-title a:hover, .site-main .post-navigation .meta-nav, .site-main .paging-navigation .meta-nav, .comment-meta .comment-metadata a:hover, .required, .widget a, #wp-calendar a, .wp_widget_tag_cloud a:hover, #infinite-footer .container a:hover, #infinite-handle span, .tags-links a:hover, .current_page_item a, .current-menu-item a, .current_page_item a:visited, .current-menu-item a:visited { color: #1B7878;}
|
||
</style>
|
||
<link rel="stylesheet" id="custom-css-css" type="text/css" href="https://s2.wp.com/?custom-css=1&csblog=1AZzO&cscache=6&csrev=16" />
|
||
</head>
|
||
|
||
<body class="wp-singular post-template-default single single-post postid-7797 single-format-standard custom-background wp-embed-responsive wp-theme-pubpenscratch-2 customizer-styles-applied no-sidebar jetpack-reblog-enabled categories-hidden author-hidden custom-colors">
|
||
<div id="page" class="hfeed site">
|
||
<a class="skip-link screen-reader-text" href="#content">Skip to content</a>
|
||
<header id="masthead" class="site-header" role="banner">
|
||
<div class="site-branding">
|
||
<p class="site-title"><a href="https://blog.acolyer.org/" rel="home">the morning paper</a></p>
|
||
<p class="site-description">a random walk through Computer Science research, by Adrian Colyer</p>
|
||
</div>
|
||
|
||
<nav id="site-navigation" class="main-navigation" role="navigation">
|
||
<button class="menu-toggle">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><rect x="0" fill="none" width="16" height="16"/><g><path id="menu-icon" d="M0 14h16v-2H0v2zM0 2v2h16V2H0zm0 7h16V7H0v2z"/></g></svg>
|
||
Menu </button>
|
||
<div class="menu"><ul>
|
||
<li class="page_item page-item-2"><a href="https://blog.acolyer.org/about/">About</a></li>
|
||
<li class="page_item page-item-8716"><a href="https://blog.acolyer.org/archives/">Archives</a></li>
|
||
<li class="page_item page-item-5027"><a href="https://blog.acolyer.org/infoq-quarterly-review-editions/">InfoQ QR Editions</a></li>
|
||
<li class="page_item page-item-8713"><a href="https://blog.acolyer.org/search/">Search</a></li>
|
||
<li class="page_item page-item-406"><a href="https://blog.acolyer.org/email-subs/">Subscribe</a></li>
|
||
<li class="page_item page-item-8719"><a href="https://blog.acolyer.org/tags/">Tags</a></li>
|
||
<li class="page_item page-item-6991"><a href="https://blog.acolyer.org/privacy/">Privacy</a></li>
|
||
</ul></div>
|
||
</nav><!-- #site-navigation -->
|
||
</header><!-- #masthead -->
|
||
|
||
<div id="content" class="site-content">
|
||
|
||
<div id="primary" class="content-area">
|
||
<main id="main" class="site-main" role="main">
|
||
|
||
|
||
|
||
<article id="post-7797" class="post-7797 post type-post status-publish format-standard hentry category-uncategorized tag-facebook tag-operations">
|
||
<header class="entry-header">
|
||
<h1 class="entry-title">Maelstrom: mitigating datacenter-level disasters by draining interdependent traffic safely and efficiently</h1>
|
||
<div class="entry-meta">
|
||
<span class="posted-on"><a href="https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/" rel="bookmark"><time class="entry-date published" datetime="2018-10-24T06:00:00+01:00">October 24, 2018</time><time class="updated" datetime="2018-10-20T07:15:40+01:00">October 20, 2018</time></a></span><span class="byline"><span class="author vcard"><span class="sep"> ~ </span><a class="url fn n" href="https://blog.acolyer.org/author/adriancolyer/">adriancolyer</a></span></span> </div><!-- .entry-meta -->
|
||
</header><!-- .entry-header -->
|
||
|
||
<div class="entry-content">
|
||
<p><a href="https://www.usenix.org/conference/osdi18/presentation/veeraraghavan">Maelstrom: mitigating datacenter-level disasters by draining interdependent traffic safely and efficiently</a> Veeraraghavan et al., <em>OSDI’18</em></p>
|
||
<p>Here’s a really valuable paper detailing four plus years of experience dealing with datacenter outages at Facebook. <em>Maelstrom</em> is the system Facebook use in production to <em>mitigate</em> and <em>recover from</em> datacenter-level disasters. The high level idea is simple: <em>drain</em> traffic away from the failed datacenter and move it to other datacenters. Doing that safely, reliably, and repeatedly, not so simple!</p>
|
||
<blockquote><p>
|
||
Modern Internet services are composed of hundreds of interdependent systems spanning dozens of geo-distributed datacenters. At this scale, seemingly rare natural disasters, such as hurricanes blowing down power lines and flooding, occur regularly.
|
||
</p></blockquote>
|
||
<p>How regularly? Well, we’re told that Maelstrom has been in production at Facebook for over four years, and in that time has helped Facebook to recover from over 100 disasters. I make that about one disaster every two weeks!! Not all of these are total loss of a datacenter, but they’re all serious datacenter-wide incidents. One example was fibrecuts leading to a loss of 85% of the backbone network capability connecting a datacenter to the FB infrastructure. Maelstrom had most user-facing traffic drained in about 17 minutes, and the all traffic drained in about 1.5 hours. Once the backbone network was repaired, Maelstrom was used to restore traffic back to the datacenter.</p>
|
||
<p>Also in that disaster count are service-level incidents caused by software errors (bugs, misconfigurations). An alert will inform operators of a serious situation, and the operators then need to decide whether to revert the offending change(s) or fix forward after diagnosing the problem. As we saw last week, <a href="https://blog.acolyer.org/2018/10/19/orca-differential-bug-localization-in-large-scale-services/">identifying the offending changes can itself take time</a>, as does coding up and testing a fix. In these circumstances, Maelstrom can reduce the impact of failures by moving diagnosis and recovery out of the critical path.</p>
|
||
<blockquote><p>
|
||
Currently, Maelstrom drains hundreds of services in a fully automated manner, with new systems being onboarded regularly. We can drain all user-facing traffic, across multiple product families, from any datacenter in less than 40 minutes.
|
||
</p></blockquote>
|
||
<p>There are three very valuable contributions in this paper:</p>
|
||
<ol>
|
||
<li>The description of Maelstrom itself, a generic framework for draining heterogeneous traffic from interdependent systems safely and efficiently. (There’s a product right there!)</li>
|
||
<li>An explanation of <em>drain tests</em> as they are practiced at Facebook: an essential component of disaster recovery.</li>
|
||
<li>A sprinkling of hard-won wisdom and expectation setting for rolling out similar capabilities in your own organisation.</li>
|
||
</ol>
|
||
<h3>Draining traffic, how hard can that really be??</h3>
|
||
<blockquote><p>
|
||
The conceptually simple idea of draining traffic turns out to be rather challenging in practice. In our experience, disasters ofter trigger failures that affect multiple interdependent systems simultaneously… we observe that most of today’s Internet services are composed of a number of heterogeneous systems including singly-homed and failover-based systems with complex, subtle dependencies and distinct traffic characteristics. The most challenging aspect of mitigation is to ensure that dependencies among systems are not violated.
|
||
</p></blockquote>
|
||
<p>And of course, you’d better not create any <em>new</em> outages or failures as a result of your draining intervention (e.g., it would be easy to cause cascading failures).</p>
|
||
<p>Based on affinity (aka ‘stickiness’) and state properties, there are four fundamental types of traffic that need to be considered during draining:</p>
|
||
<p><img src="https://blog.acolyer.org/wp-content/uploads/2018/10/maelstrom-table-1.jpeg?w=480" alt="" /></p>
|
||
<ul>
|
||
<li>Stateless traffic (e.g., the majority of web traffic) is easy to deal with.</li>
|
||
<li>Sticky traffic (e.g. messaging) pins to particular machines maintaining session state. For sticky traffic incoming session requests need to be rerouted, and established session torn down.</li>
|
||
<li>Replication traffic rerouting typically requires configuration changes or other heavyweight changes.</li>
|
||
<li>Stateful traffic (e.g. hot standbys) typically requires promoting a secondary to become the new master. State may need to be carefully copied from one datacenter to another.</li>
|
||
</ul>
|
||
<p>All of the above needs to be done in the right order as well, based on the dependency graph.</p>
|
||
<h3>Introducing Maelstrom</h3>
|
||
<p>Maelstrom is the system that Facebook use to execute disaster mitigation and recovery <em>runbooks</em>.</p>
|
||
<blockquote><p>
|
||
… a runbook specified the concrete procedure for mitigating the particular disaster scenario by draining traffic out of the datacenter; after the root causes are resolved, a corresponding recovery runbook is used to restore the traffic back…
|
||
</p></blockquote>
|
||
<p>Runbooks are made up of <em>tasks</em> (e.g. shifting a portion of traffic, migrating data shards, etc.) which can have <em>dependencies</em> between them. The Maelstrom UI can be used to launch runbooks, and monitor their progress.</p>
|
||
<p><img src="https://blog.acolyer.org/wp-content/uploads/2018/10/maelstrom-fig-2.jpeg?w=640" alt="" /></p>
|
||
<p>Every service maintains its own <em>service-specific</em> runbook for disaster mitigation. If an entire datacenter is down, a <em>datacenter evacuation runbook</em> is used to drain the traffic of all the services in the datacenter. Such a runbook aggregates a collection of service-specific runbooks.</p>
|
||
<p><img src="https://blog.acolyer.org/wp-content/uploads/2018/10/maelstrom-fig-11.jpeg?w=520" alt="" /></p>
|
||
<p>Tasks are defined based on a collection of task templates. Each task has associated health metrics, pre-conditions and post-conditions, and dependencies.</p>
|
||
<blockquote><p>
|
||
We implement a library of stages that capture common operations like instructing load balancers to alter traffic allocation, managing containers and jobs, changing system configuration, migrating data shards, etc.
|
||
</p></blockquote>
|
||
<p>The Maelstrom runtime engine comprises a scheduler that determines the correct order of task execution and an executor that executes tasks and validates the results. Maelstrom compares the duration of each task to the 75th percentile of the execution time of previous runs to determine whether a task is stuck in execution. If so, an operator is alerted. “<em>We prioritize safety over speed, and stall subsequent operations until an operator intervenes</em>“. During execution, Maelstrom uses a closed feedback loop to pace the speed of traffic drains based on extensive health monitoring. After executing a runbook, Maelstrom performs automated critical path analysis to help optimise mitigation time by identifying bottlenecks in the dependency graphs.</p>
|
||
<h3>Drain testing</h3>
|
||
<p>Maelstrom can only be as good as its runbooks…</p>
|
||
<blockquote><p>
|
||
Maelstrom requires runbooks to always keep updated with our rapidly-evolving software systems and physical infrastructure. However, maintaining up-to-date information (e.g. service dependencies) is challenging due to the complexity and dynamics of systems at scale… drain tests are our practical solution to continuously verify and build trust in the runbooks.
|
||
</p></blockquote>
|
||
<p>Drain tests are fully automated tests that use Maelstrom to drain user-facing and internal service traffic away from a datacenter, just as if that datacenter was failing. Multiple drain tests are run per week to simulate various disaster scenarios, with tests being scheduled at various times to cover different traffic patterns. In addition to regular drain tests, FB also run <em>storm</em> tests at a quarterly cadence. Storm tests evaluate the ability to recover from the total loss of a datacenter, and are similar to <a href="https://queue.acm.org/detail.cfm?id=2371516">Google’s DiRT</a> and <a href="https://queue.acm.org/detail.cfm?id=2371297">Amazon’s GameDay</a> exercises.</p>
|
||
<p>Running regular drain tests is the only way to know that the runbooks will do their job when called upon in a real emergency. They operate on live production traffic and are not expected to have any user-visible or service-lever impact: if they do, a follow-up with the engineering team investigates why a given disaster scenario was not handled well.</p>
|
||
<blockquote><p>
|
||
Drain tests also force us to gain a deep understanding of our complex, dynamic systems and infrastructure, and help us plan capacity for projected demand, audit utilization of shared resources, and discover dependencies.
|
||
</p></blockquote>
|
||
<p>Drain tests are an important part of verifying that the shared infrastructure has sufficient capacity to absorb spikes in utilization caused by draining with Maelstrom. If a test identifies a bottleneck, “<em>we work with teams to update our routing policies, traffic tagging and prioritization schemes, or bandwidth reservation configuration so we can drain services safely.</em>“. In addition to capacity planning, <em>traffic prioritization</em> is used to drain user-facing traffic as soon as possible, followed by stateful service traffic. In the event of resource overload, many systems also employ <a href="https://blog.acolyer.org/2014/10/24/brownout-building-more-robust-cloud-applications/">graceful degradation with PID controllers</a> (e.g., incrementally turning off ranking algorithm complexity to reduce compute demands).</p>
|
||
<h3>Creating runbooks and onboarding services</h3>
|
||
<p>Drain tests help ensure that runbooks are working as desired (not always the same thing as working-as-designed ;) ). One of the main challenges when onboarding a new service is discovering all of its dependencies. Three common dependency relationships often manifest:</p>
|
||
<ul>
|
||
<li><em>Bootstrapping</em> dependencies occur when a service depends on other components to prepare its execution environment and setup configuration before serving traffic.</li>
|
||
<li><em>RPC</em> dependencies occur when a service makes RPCs to fetch data from other services during its operation</li>
|
||
<li><em>State</em> dependencies occur when depending upon traffic state across services, e.g. a proxy to coordinate and manage session establishment.</li>
|
||
</ul>
|
||
<p>Uncovering dependencies starts with a scenario-driven questionnaire coupled with an analysis of the outputs of tracing and service discovery systems. Clues to state-based dependencies can also often be found in logs. Once dependencies are identified, they are organised into chains of parent-child dependencies in a disconnected dependency graph. Then common services are identified across chains, these often indicate <em>highly connected components</em> (HCCs).</p>
|
||
<p><img src="https://blog.acolyer.org/wp-content/uploads/2018/10/maelstrom-fig-5.jpeg?w=520" alt="" /></p>
|
||
<blockquote><p>
|
||
Draining a HCC service will likely require us to drain its parents first; once the HCC service is drained, its children can likely be drained concurrently.
|
||
</p></blockquote>
|
||
<p>Once the initial runbook has been constructed, Maelstrom is used to cautiously validate it via small-scale drain tests, with careful monitoring of the health of all systems involved. The radius of the drain tests is gradually enlarged until all levels of traffic drains can be performed regularly.</p>
|
||
<h3>Lessons learned</h3>
|
||
<ul>
|
||
<li>Drain tests help us understand interactions amongst systems in our complex infrastructure.</li>
|
||
<li>Drain tests help us prepare for disaster</li>
|
||
<li>Drain tests are challenging to run</li>
|
||
<li>Automating disaster mitigation completely is not a goal. The original vision was to take humans out of the loop altogether. The current design is focused around helping operators triage a disaster and efficiently mitigate it.</li>
|
||
<li>Building the right abstractions to handle failures is important, but takes time and iteration. The separation of runbooks and tasks allows each team to focus on their service specific policies without the need to rebuild mechanisms. Sometimes onboarding a new service does require the creation of new task templates.</li>
|
||
<li>It takes time to reach maturity:</li>
|
||
</ul>
|
||
<blockquote><p>
|
||
Our original tests only targeted one stateless system: our web servers. The first set of drain tests were painful – they took more than 10 hours, and experienced numerous interruptions as we uncovered dependencies or triggered failures that resulted in service-level issues…. after a year we extended drain tests to two more services: a photo sharing service and a real-time messaging service.
|
||
</p></blockquote>
|
||
<p>Four-and-a-bit years in, and Maelstrom can now drain hundreds of services.</p>
|
||
<h3>Maelstrom in action.</h3>
|
||
<p>Section 5 in the paper contains many examples of Maelstrom draining different types of traffic for Facebook services. I’m already over target length so won’t cover it here, but do check out the full paper for details if this topic interests you.</p>
|
||
<h3>The last word</h3>
|
||
<blockquote><p>
|
||
As our infrastructure grows, we have learned that it is critical to develop trusted tools and mechanisms to prepare for and respond to failure… Much of the focus of Maelstrom has been around ensuring that Facebook stays available when an incident affects an entire datacenter. In practice, we find that many incidents affect only a subset of hardware and software systems rather than entire datacenters. Our next focus is on building tools to isolate outages to the minimal subset of the systems they affect.
|
||
</p></blockquote>
|
||
<div id="jp-post-flair" class="sharedaddy sd-like-enabled sd-sharing-enabled"><div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-twitter"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-twitter-7797"
|
||
class="share-twitter sd-button share-icon"
|
||
href="https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/?share=twitter"
|
||
target="_blank"
|
||
aria-labelledby="sharing-twitter-7797"
|
||
>
|
||
<span id="sharing-twitter-7797" hidden>Share on X (Opens in new window)</span>
|
||
<span>X</span>
|
||
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-linkedin-7797"
|
||
class="share-linkedin sd-button share-icon"
|
||
href="https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/?share=linkedin"
|
||
target="_blank"
|
||
aria-labelledby="sharing-linkedin-7797"
|
||
>
|
||
<span id="sharing-linkedin-7797" hidden>Share on LinkedIn (Opens in new window)</span>
|
||
<span>LinkedIn</span>
|
||
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-email-7797"
|
||
class="share-email sd-button share-icon"
|
||
href="mailto:?subject=%5BShared%20Post%5D%20Maelstrom%3A%20mitigating%20datacenter-level%20disasters%20by%20draining%20interdependent%20traffic%20safely%20and%20efficiently&body=https%3A%2F%2Fblog.acolyer.org%2F2018%2F10%2F24%2Fmaelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently%2F&share=email"
|
||
target="_blank"
|
||
aria-labelledby="sharing-email-7797"
|
||
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you're having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="2aa24d10c1" data-email-share-track-url="https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/?share=email">
|
||
<span id="sharing-email-7797" hidden>Email a link to a friend (Opens in new window)</span>
|
||
<span>Email</span>
|
||
</a></li><li class="share-print"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-print-7797"
|
||
class="share-print sd-button share-icon"
|
||
href="https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/#print?share=print"
|
||
target="_blank"
|
||
aria-labelledby="sharing-print-7797"
|
||
>
|
||
<span id="sharing-print-7797" hidden>Print (Opens in new window)</span>
|
||
<span>Print</span>
|
||
</a></li><li class="share-end"></li></ul></div></div></div><div class='sharedaddy sd-block sd-like jetpack-likes-widget-wrapper jetpack-likes-widget-unloaded' id='like-post-wrapper-23592848-7797-6aa4938f0d68d' data-src='//widgets.wp.com/likes/index.html?ver=20260911#blog_id=23592848&post_id=7797&origin=adriancolyer.wordpress.com&obj_id=23592848-7797-6aa4938f0d68d&domain=blog.acolyer.org' data-name='like-post-frame-23592848-7797-6aa4938f0d68d' data-title='Like or Reblog'><div class='likes-widget-placeholder post-likes-widget-placeholder' style='height: 55px;'><span class='button'><span>Like</span></span> <span class='loading'>Loading...</span></div><span class='sd-text-color'></span><a class='sd-link-color'></a></div>
|
||
<div id='jp-relatedposts' class='jp-relatedposts' >
|
||
<h3 class="jp-relatedposts-headline"><em>Related</em></h3>
|
||
</div></div> </div><!-- .entry-content -->
|
||
|
||
<footer class="entry-footer">
|
||
<span class="cat-links">
|
||
Posted in <a href="https://blog.acolyer.org/category/uncategorized/" rel="category tag">Uncategorized</a> </span>
|
||
<span class="tags-links">
|
||
<a href="https://blog.acolyer.org/tag/facebook/" rel="tag">Facebook</a><a href="https://blog.acolyer.org/tag/operations/" rel="tag">Operations</a> </span>
|
||
</footer><!-- .entry-footer -->
|
||
|
||
</article><!-- #post-## -->
|
||
|
||
|
||
<nav class="navigation post-navigation" aria-label="Posts">
|
||
<h2 class="screen-reader-text">Post navigation</h2>
|
||
<div class="nav-links"><div class="nav-previous"><a href="https://blog.acolyer.org/2018/10/22/legoos-a-disseminated-distributed-os-for-hardware-resource-disaggregation/" rel="prev"><span class="meta-nav">‹ Previous</span>LegoOS: a disseminated, distributed OS for hardware resource disaggregation</a></div><div class="nav-next"><a href="https://blog.acolyer.org/2018/10/26/robinhood-tail-latency-aware-caching-dynamic-reallocation-from-cache-rich-to-cache-poor/" rel="next"><span class="meta-nav">Next ›</span>RobinHood: tail latency aware caching – dynamic reallocation from cache-rich to cache-poor</a></div></div>
|
||
</nav>
|
||
|
||
<div id="comments" class="comments-area">
|
||
|
||
|
||
<h2 class="comments-title">
|
||
4 thoughts on “<span>Maelstrom: mitigating datacenter-level disasters by draining interdependent traffic safely and efficiently</span>” </h2><!-- .comments-title -->
|
||
|
||
|
||
<ol class="comment-list">
|
||
<li id="comment-33405" class="comment even thread-even depth-1">
|
||
<article id="div-comment-33405" class="comment-body">
|
||
<footer class="comment-meta">
|
||
<div class="comment-author vcard">
|
||
<img referrerpolicy="no-referrer" alt='steffen's avatar' src='https://1.gravatar.com/avatar/dd443b353b45eb06fe848e395898fefa6fd8b16ddcabf28cda55148396bce8ef?s=60&d=identicon&r=G' srcset='https://1.gravatar.com/avatar/dd443b353b45eb06fe848e395898fefa6fd8b16ddcabf28cda55148396bce8ef?s=60&d=identicon&r=G 1x, https://1.gravatar.com/avatar/dd443b353b45eb06fe848e395898fefa6fd8b16ddcabf28cda55148396bce8ef?s=90&d=identicon&r=G 1.5x, https://1.gravatar.com/avatar/dd443b353b45eb06fe848e395898fefa6fd8b16ddcabf28cda55148396bce8ef?s=120&d=identicon&r=G 2x, https://1.gravatar.com/avatar/dd443b353b45eb06fe848e395898fefa6fd8b16ddcabf28cda55148396bce8ef?s=180&d=identicon&r=G 3x, https://1.gravatar.com/avatar/dd443b353b45eb06fe848e395898fefa6fd8b16ddcabf28cda55148396bce8ef?s=240&d=identicon&r=G 4x' class='avatar avatar-60' height='60' width='60' decoding='async' /> <b class="fn">steffen</b> <span class="says">says:</span> </div><!-- .comment-author -->
|
||
|
||
<div class="comment-metadata">
|
||
<a href="https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/#comment-33405"><time datetime="2018-10-24T10:02:19+01:00">October 24, 2018 at 10:02 am</time></a> </div><!-- .comment-metadata -->
|
||
|
||
</footer><!-- .comment-meta -->
|
||
|
||
<div class="comment-content">
|
||
<p>Sounds to me like “we do not have and are unable to develop distributed applications, and they’re just too many of them, so we put some scripts in place to automate moving and restarting of those tons of services. since nobody cares to properly architect and define dependencies we run regular crash tests and fix the scripts as we go.”</p>
|
||
<p>I’m not an expert in distributed systems but I wonder: is the state-of-art really that we cannot build these things? Or is it an artifact of starting out with 1 server and simply not having the necessary developer skill and tools in place..?</p>
|
||
</div><!-- .comment-content -->
|
||
|
||
</article><!-- .comment-body -->
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-34261" class="pingback odd alt thread-odd thread-alt depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://devzen.ru/episode-0216/" class="url" rel="ugc external nofollow">Консенсус программного комитета — Episode 0216 « DevZen Podcast</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-34290" class="pingback even thread-even depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://blog.acolyer.org/2018/11/05/sharding-the-shards-managing-datastore-locality-at-scale-with-akkio/" class="url" rel="ugc">Sharding the shards: managing datastore locality at scale with Akkio | the morning paper</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-34348" class="pingback odd alt thread-odd thread-alt depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://daily-ai.com/sharding-the-shards-managing-datastore-locality-at-scale-with-akkio/" class="url" rel="ugc external nofollow">Sharding the shards: managing datastore locality at scale with Akkio | Daily AI</a> </div>
|
||
</li><!-- #comment-## -->
|
||
</ol><!-- .comment-list -->
|
||
|
||
|
||
|
||
<p class="no-comments">Comments are closed.</p>
|
||
|
||
|
||
</div><!-- #comments -->
|
||
|
||
|
||
</main><!-- #main -->
|
||
</div><!-- #primary -->
|
||
|
||
|
||
</div><!-- #content -->
|
||
|
||
<footer id="colophon" class="site-footer" role="contentinfo">
|
||
<div class="site-info-wrapper clear">
|
||
<div class="site-info">
|
||
<a href="https://wordpress.com/?ref=footer_blog" rel="nofollow">Blog at WordPress.com.</a>
|
||
|
||
</div><!-- .site-info -->
|
||
</div><!-- .site-info-wrapper -->
|
||
</footer><!-- #colophon -->
|
||
</div><!-- #page -->
|
||
|
||
<!-- -->
|
||
<script type="speculationrules">
|
||
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/files/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/pub/penscratch-2/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
|
||
</script>
|
||
<script id="grofiles-cards-js" src="//0.gravatar.com/js/hovercards/hovercards.min.js?ver=202637d1133547e8a1f2801609f51cef12522abe7f6647d869e9fbc974fe356a37e613"></script>
|
||
<script id="wpgroho-js-extra">
|
||
var WPGroHo = {"my_hash":""};
|
||
//# sourceURL=wpgroho-js-extra
|
||
</script>
|
||
<script crossorigin='anonymous' type='text/javascript' src='/wp-content/mu-plugins/gravatar-hovercards/wpgroho.js?m=1610363240i'></script>
|
||
|
||
<script>
|
||
// Initialize and attach hovercards to all gravatars
|
||
( function() {
|
||
function init() {
|
||
if ( typeof Gravatar === 'undefined' ) {
|
||
return;
|
||
}
|
||
|
||
if ( typeof Gravatar.init !== 'function' ) {
|
||
return;
|
||
}
|
||
|
||
Gravatar.profile_cb = function ( hash, id ) {
|
||
WPGroHo.syncProfileData( hash, id );
|
||
};
|
||
|
||
Gravatar.my_hash = WPGroHo.my_hash;
|
||
Gravatar.init(
|
||
'body',
|
||
'#wp-admin-bar-my-account',
|
||
{
|
||
i18n: {
|
||
'Edit your profile →': 'Edit your profile →',
|
||
'View profile →': 'View profile →',
|
||
'Contact': 'Contact',
|
||
'Send money': 'Send money',
|
||
'Sorry, we are unable to load this Gravatar profile.': 'Sorry, we are unable to load this Gravatar profile.',
|
||
'Gravatar not found.': 'Gravatar not found.',
|
||
'Too Many Requests.': 'Too Many Requests.',
|
||
'Internal Server Error.': 'Internal Server Error.',
|
||
'Is this you?': 'Is this you?',
|
||
'Claim your free profile.': 'Claim your free profile.',
|
||
'Email': 'Email',
|
||
'Home Phone': 'Home Phone',
|
||
'Work Phone': 'Work Phone',
|
||
'Cell Phone': 'Cell Phone',
|
||
'Contact Form': 'Contact Form',
|
||
'Calendar': 'Calendar',
|
||
},
|
||
}
|
||
);
|
||
}
|
||
|
||
if ( document.readyState !== 'loading' ) {
|
||
init();
|
||
} else {
|
||
document.addEventListener( 'DOMContentLoaded', init );
|
||
}
|
||
} )();
|
||
</script>
|
||
|
||
<div style="display:none">
|
||
<div class="grofile-hash-map-2a3adab8bf77de714450f3bbba69cdaf">
|
||
</div>
|
||
</div>
|
||
<div id="actionbar" dir="ltr" style="display: none;"
|
||
class="actnbr-pub-penscratch-2 actnbr-has-follow actnbr-has-actions">
|
||
<ul>
|
||
<li class="actnbr-btn actnbr-hidden">
|
||
<a class="actnbr-action actnbr-actn-reblog" href="">
|
||
<svg class="gridicon gridicons-reblog" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M22.086 9.914L20 7.828V18c0 1.105-.895 2-2 2h-7v-2h7V7.828l-2.086 2.086L14.5 8.5 19 4l4.5 4.5-1.414 1.414zM6 16.172V6h7V4H6c-1.105 0-2 .895-2 2v10.172l-2.086-2.086L.5 15.5 5 20l4.5-4.5-1.414-1.414L6 16.172z"/></g></svg><span>Reblog</span>
|
||
</a>
|
||
</li>
|
||
<li class="actnbr-btn actnbr-hidden">
|
||
<a class="actnbr-action actnbr-actn-follow " href="">
|
||
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
|
||
<span>Subscribe</span>
|
||
</a>
|
||
<a class="actnbr-action actnbr-actn-following no-display" href="">
|
||
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
|
||
<span>Subscribed</span>
|
||
</a>
|
||
<div class="actnbr-popover tip tip-top-left actnbr-notice" id="follow-bubble">
|
||
<div class="tip-arrow"></div>
|
||
<div class="tip-inner actnbr-follow-bubble">
|
||
<ul>
|
||
<li class="actnbr-sitename">
|
||
<a href="https://blog.acolyer.org">
|
||
<img loading='lazy' alt='' src='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png' srcset='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=75&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1.5x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=100&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 2x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=150&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 3x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=200&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 4x' class='avatar avatar-50' height='50' width='50' /> the morning paper </a>
|
||
</li>
|
||
<div class="actnbr-message no-display"></div>
|
||
<form method="post" action="https://subscribe.wordpress.com" accept-charset="utf-8" style="display: none;">
|
||
<div class="actnbr-follow-count">Join 3,125 other subscribers</div>
|
||
<div>
|
||
<input type="email" name="email" placeholder="Enter your email address" class="actnbr-email-field" aria-label="Enter your email address" />
|
||
</div>
|
||
<input type="hidden" name="action" value="subscribe" />
|
||
<input type="hidden" name="blog_id" value="23592848" />
|
||
<input type="hidden" name="source" value="https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/" />
|
||
<input type="hidden" name="sub-type" value="actionbar-follow" />
|
||
<input type="hidden" id="_wpnonce" name="_wpnonce" value="4c8bf0f483" /> <div class="actnbr-button-wrap">
|
||
<button type="submit" value="Sign me up">
|
||
Sign me up </button>
|
||
</div>
|
||
</form>
|
||
<li class="actnbr-login-nudge">
|
||
<div>
|
||
Already have a WordPress.com account? <a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fblog.acolyer.org%252F2018%252F10%252F24%252Fmaelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently%252F">Log in now.</a> </div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</li>
|
||
<li class="actnbr-ellipsis actnbr-hidden">
|
||
<svg class="gridicon gridicons-ellipsis" height="24" width="24" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M7 12c0 1.104-.896 2-2 2s-2-.896-2-2 .896-2 2-2 2 .896 2 2zm12-2c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2zm-7 0c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2z"/></g></svg> <div class="actnbr-popover tip tip-top-left actnbr-more">
|
||
<div class="tip-arrow"></div>
|
||
<div class="tip-inner">
|
||
<ul>
|
||
<li class="actnbr-sitename">
|
||
<a href="https://blog.acolyer.org">
|
||
<img loading='lazy' alt='' src='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png' srcset='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=75&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1.5x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=100&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 2x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=150&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 3x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=200&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 4x' class='avatar avatar-50' height='50' width='50' /> the morning paper </a>
|
||
</li>
|
||
<li class="actnbr-folded-follow">
|
||
<a class="actnbr-action actnbr-actn-follow " href="">
|
||
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
|
||
<span>Subscribe</span>
|
||
</a>
|
||
<a class="actnbr-action actnbr-actn-following no-display" href="">
|
||
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
|
||
<span>Subscribed</span>
|
||
</a>
|
||
</li>
|
||
<li class="actnbr-signup"><a href="https://wordpress.com/start/">Sign up</a></li>
|
||
<li class="actnbr-login"><a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fblog.acolyer.org%252F2018%252F10%252F24%252Fmaelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently%252F">Log in</a></li>
|
||
<li class="actnbr-shortlink">
|
||
<a href="https://wp.me/p1AZzO-21L">
|
||
<span class="actnbr-shortlink__text">Copy shortlink</span>
|
||
<span class="actnbr-shortlink__icon"><svg class="gridicon gridicons-checkmark" height="16" width="16" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M9 19.414l-6.707-6.707 1.414-1.414L9 16.586 20.293 5.293l1.414 1.414"/></g></svg></span>
|
||
</a>
|
||
</li>
|
||
<li class="flb-report">
|
||
<a href="https://wordpress.com/abuse/?report_url=https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/" target="_blank" rel="noopener noreferrer">
|
||
Report this content </a>
|
||
</li>
|
||
<li class="actnbr-reader">
|
||
<a href="https://wordpress.com/reader/blogs/23592848/posts/7797">
|
||
View post in Reader </a>
|
||
</li>
|
||
<li class="actnbr-subs">
|
||
<a href="https://subscribe.wordpress.com/">Manage subscriptions</a>
|
||
</li>
|
||
<li class="actnbr-fold"><a href="">Collapse this bar</a></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
<script>
|
||
window.addEventListener( "DOMContentLoaded", function( event ) {
|
||
var link = document.createElement( "link" );
|
||
link.href = "/wp-content/mu-plugins/actionbar/actionbar.css?v=20250116";
|
||
link.type = "text/css";
|
||
link.rel = "stylesheet";
|
||
document.head.appendChild( link );
|
||
|
||
var script = document.createElement( "script" );
|
||
script.src = "/wp-content/mu-plugins/actionbar/actionbar.js?v=20250204";
|
||
document.body.appendChild( script );
|
||
} );
|
||
</script>
|
||
|
||
|
||
<script type="text/javascript">
|
||
window.WPCOM_sharing_counts = {"https://blog.acolyer.org/2018/10/24/maelstrom-mitigating-datacenter-level-disasters-by-draining-interdependent-traffic-safely-and-efficiently/":7797};
|
||
</script>
|
||
|
||
<script type="text/javascript">
|
||
(function () {
|
||
var wpcom_reblog = {
|
||
source: 'toolbar',
|
||
|
||
toggle_reblog_box_flair: function (obj_id, post_id) {
|
||
|
||
// Go to site selector. This will redirect to their blog if they only have one.
|
||
const postEndpoint = `https://wordpress.com/post`;
|
||
|
||
// Ideally we would use the permalink here, but fortunately this will be replaced with the
|
||
// post permalink in the editor.
|
||
const originalURL = `${ document.location.href }?page_id=${ post_id }`;
|
||
|
||
const url =
|
||
postEndpoint +
|
||
'?url=' +
|
||
encodeURIComponent( originalURL ) +
|
||
'&is_post_share=true' +
|
||
'&v=5';
|
||
|
||
const redirect = function () {
|
||
if (
|
||
! window.open( url, '_blank' )
|
||
) {
|
||
location.href = url;
|
||
}
|
||
};
|
||
|
||
if ( /Firefox/.test( navigator.userAgent ) ) {
|
||
setTimeout( redirect, 0 );
|
||
} else {
|
||
redirect();
|
||
}
|
||
},
|
||
};
|
||
|
||
window.wpcom_reblog = wpcom_reblog;
|
||
})();
|
||
</script>
|
||
<script id="sharing-js-js-extra">
|
||
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":"1"};
|
||
//# sourceURL=sharing-js-js-extra
|
||
</script>
|
||
<script crossorigin='anonymous' type='text/javascript' src='/_static/??-eJyVj0kOwjAQBD+EM4BEEg6IpyDHnsTjZWxim+X3LOIQccutS6o6NNyTUJELcoFQRfJ1Is7gyWGGa8WKRrL2ODc2b2AhF4PhraQ6QELOapZFGbEHm4HljSZZKPKKaMkrsuwoCU/sxBhVzWKkx3+9uGWxJKncjyHEyHAhVjBU8hqykTNqqfXzO4mnJtDnxDmcdl3fHQ/bvm3tC5SKcX4='></script>
|
||
<script id="sharing-js-js-after">
|
||
window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
|
||
window.WPCOM_sharing_popups[ "twitter" ] = "menubar=1,resizable=1,width=600,height=350";
|
||
window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
|
||
window.WPCOM_sharing_popups[ "linkedin" ] = "menubar=1,resizable=1,width=580,height=450";
|
||
//# sourceURL=sharing-js-js-after
|
||
</script>
|
||
<script id="wp-emoji-settings" type="application/json">
|
||
{"baseUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/72x72/","ext":".png","svgUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/svg/","svgExt":".svg","source":{"concatemoji":"/wp-includes/js/wp-emoji-release.min.js?m=1783090810i&ver=7.1.1-alpha-63555"}}
|
||
</script>
|
||
<script type="module">
|
||
/*! This file is auto-generated */
|
||
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
|
||
//# sourceURL=/wp-includes/js/wp-emoji-loader.min.js
|
||
</script>
|
||
<iframe src='https://widgets.wp.com/likes/master.html?ver=20260911#ver=20260911' scrolling='no' id='likes-master' name='likes-master' style='display:none;'></iframe>
|
||
<div id='likes-other-gravatars' role="dialog" aria-hidden="true" tabindex="-1"><div class="likes-text"><span>%d</span></div><ul class="wpl-avatars sd-like-gravatars"></ul></div>
|
||
<script src="//stats.wp.com/w.js?69" defer></script> <script type="text/javascript">
|
||
_tkq = window._tkq || [];
|
||
_stq = window._stq || [];
|
||
_tkq.push(['storeContext', {'blog_id':'23592848','blog_tz':'1','user_lang':'en','blog_lang':'en','user_id':'0'}]);
|
||
// Prevent sending pageview tracking from WP-Admin pages.
|
||
_stq.push(['view', {'blog':'23592848','v':'wpcom','tz':'1','user_id':'0','post':'7797','subd':'adriancolyer'}]);
|
||
_stq.push(['extra', {'crypt':'UE5tW3cvZGRDbzJxMWx6aVlydldVeXIvdUpWanJmOVg4Mys2XWxoN19+Lk5YJUp6QyVxQW1tPTUzTXBXJWFTeGhyW0lEYUpXRVFiZHUvZnJpUCVPVy1fVltFNCZ4fFNoLkJhdXVsWkhsP0NDM3FBeCxqN21iQWc/eUVmbkgwWVRQaj9lPzlHdmpWdVhmWUNWV3ZsU2xFRjMlUFN6alk4UXB4RVp0OFQ3ZExuL0N1PUJZSEZTd35rcmNEQmg2OF83flNRLCxaQmZqdjNydGEzLC5FTVBjalZwSVJGUT1dcT8='}]);
|
||
_stq.push([ 'clickTrackerInit', '23592848', '7797' ]);
|
||
</script>
|
||
<noscript><img src="https://pixel.wp.com/b.gif?v=noscript" style="height:1px;width:1px;overflow:hidden;position:absolute;bottom:1px;" alt="" /></noscript>
|
||
<meta id="bilmur" property="bilmur:data" content="" data-provider="wordpress.com" data-service="simple" data-site-tz="Europe/London" data-custom-props="{"logged_in":"0","wptheme":"pub\/penscratch-2","wptheme_is_block":"0"}" >
|
||
<script defer src="/wp-content/js/bilmur.min.js?i=17&m=202637"></script>
|
||
</body>
|
||
</html>
|