Files
nexus/sreweekly/articles/176/05-automating-chaos-experiments-in-production.html
2026-09-12 17:23:01 +08:00

916 lines
96 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="profile" href="http://gmpg.org/xfn/11">
<title>Automating chaos experiments in production &#8211; the morning paper</title>
<meta name='robots' content='max-image-preview:large' />
<!-- Async WordPress.com Remote Login -->
<script id="wpcom_remote_login_js">
var wpcom_remote_login_extra_auth = '';
function wpcom_remote_login_remove_dom_node_id( element_id ) {
var dom_node = document.getElementById( element_id );
if ( dom_node ) { dom_node.parentNode.removeChild( dom_node ); }
}
function wpcom_remote_login_remove_dom_node_classes( class_name ) {
var dom_nodes = document.querySelectorAll( '.' + class_name );
for ( var i = 0; i < dom_nodes.length; i++ ) {
dom_nodes[ i ].parentNode.removeChild( dom_nodes[ i ] );
}
}
function wpcom_remote_login_final_cleanup() {
wpcom_remote_login_remove_dom_node_classes( "wpcom_remote_login_msg" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_validate" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_js" );
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_iframe" );
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_styles" );
}
// Watch for messages back from the remote login
window.addEventListener( "message", function( e ) {
if ( e.origin === "https://r-login.wordpress.com" ) {
var data = {};
try {
data = JSON.parse( e.data );
} catch( e ) {
wpcom_remote_login_final_cleanup();
return;
}
if ( data.msg === 'LOGIN' ) {
// Clean up the login check iframe
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
var id_regex = new RegExp( /^[0-9]+$/ );
var token_regex = new RegExp( /^.*|.*|.*$/ );
if (
token_regex.test( data.token )
&& id_regex.test( data.wpcomid )
) {
// We have everything we need to ask for a login
var script = document.createElement( "script" );
script.setAttribute( "id", "wpcom_remote_login_validate" );
script.src = '/remote-login.php?wpcom_remote_login=validate'
+ '&wpcomid=' + data.wpcomid
+ '&token=' + encodeURIComponent( data.token )
+ '&host=' + window.location.protocol
+ '//' + window.location.hostname
+ '&postid=8869'
+ '&is_singular=1';
document.body.appendChild( script );
}
return;
}
// Safari ITP, not logged in, so redirect
if ( data.msg === 'LOGIN-REDIRECT' ) {
window.location = 'https://wordpress.com/log-in?redirect_to=' + window.location.href;
return;
}
// Safari ITP, storage access failed, remove the request
if ( data.msg === 'LOGIN-REMOVE' ) {
var css_zap = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } @media screen and ( max-width: 782px ) { html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } }';
var style_zap = document.createElement( 'style' );
style_zap.type = 'text/css';
style_zap.appendChild( document.createTextNode( css_zap ) );
document.body.appendChild( style_zap );
var e = document.getElementById( 'wpcom_request_access_iframe' );
e.parentNode.removeChild( e );
document.cookie = 'wordpress_com_login_access=denied; path=/; max-age=31536000';
return;
}
// Safari ITP
if ( data.msg === 'REQUEST_ACCESS' ) {
console.log( 'request access: safari' );
// Check ITP iframe enable/disable knob
if ( wpcom_remote_login_extra_auth !== 'safari_itp_iframe' ) {
return;
}
// If we are in a "private window" there is no ITP.
var private_window = false;
try {
var opendb = window.openDatabase( null, null, null, null );
} catch( e ) {
private_window = true;
}
if ( private_window ) {
console.log( 'private window' );
return;
}
var iframe = document.createElement( 'iframe' );
iframe.id = 'wpcom_request_access_iframe';
iframe.setAttribute( 'scrolling', 'no' );
iframe.setAttribute( 'sandbox', 'allow-storage-access-by-user-activation allow-scripts allow-same-origin allow-top-navigation-by-user-activation' );
iframe.src = 'https://r-login.wordpress.com/remote-login.php?wpcom_remote_login=request_access&origin=' + encodeURIComponent( data.origin ) + '&wpcomid=' + encodeURIComponent( data.wpcomid );
var css = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 46px !important; } * html body { margin-top: 46px !important; } @media screen and ( max-width: 660px ) { html { margin-top: 71px !important; } * html body { margin-top: 71px !important; } #wpcom_request_access_iframe { display: block; height: 71px !important; } } #wpcom_request_access_iframe { border: 0px; height: 46px; position: fixed; top: 0; left: 0; width: 100%; min-width: 100%; z-index: 99999; background: #23282d; } ';
var style = document.createElement( 'style' );
style.type = 'text/css';
style.id = 'wpcom_request_access_styles';
style.appendChild( document.createTextNode( css ) );
document.body.appendChild( style );
document.body.appendChild( iframe );
}
if ( data.msg === 'DONE' ) {
wpcom_remote_login_final_cleanup();
}
}
}, false );
// Inject the remote login iframe after the page has had a chance to load
// more critical resources
window.addEventListener( "DOMContentLoaded", function( e ) {
var iframe = document.createElement( "iframe" );
iframe.style.display = "none";
iframe.setAttribute( "scrolling", "no" );
iframe.setAttribute( "id", "wpcom_remote_login_key" );
iframe.src = "https://r-login.wordpress.com/remote-login.php"
+ "?wpcom_remote_login=key"
+ "&origin=aHR0cHM6Ly9ibG9nLmFjb2x5ZXIub3Jn"
+ "&wpcomid=23592848"
+ "&time=" + Math.floor( Date.now() / 1000 );
document.body.appendChild( iframe );
}, false );
</script>
<link rel='dns-prefetch' href='//s2.wp.com' />
<link rel='dns-prefetch' href='//fonts-api.wp.com' />
<link rel='dns-prefetch' href='//s1.wp.com' />
<link rel='dns-prefetch' href='//s0.wp.com' />
<link rel="alternate" type="application/rss+xml" title="the morning paper &raquo; Feed" href="https://blog.acolyer.org/feed/" />
<link rel="alternate" type="application/rss+xml" title="the morning paper &raquo; Comments Feed" href="https://blog.acolyer.org/comments/feed/" />
<link rel="alternate" type="application/rss+xml" title="the morning paper &raquo; Automating chaos experiments in&nbsp;production Comments Feed" href="https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/feed/" />
<script type="text/javascript">
/* <![CDATA[ */
function addLoadEvent(func) {
var oldonload = window.onload;
if (typeof window.onload != 'function') {
window.onload = func;
} else {
window.onload = function () {
oldonload();
func();
}
}
}
/* ]]> */
</script>
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-2-1' href='/_static/??-eJxtjEsKgDAMBS9kjbqwuBDPUtog9o9J8fpaEQSRBw8GhoEjC50iY2QIRWRf1i0SWOSstHsYQkr1TPFIsKNXjEbkRPyhVhM18J/0m8M3fFPVlzD3UnbTtWG0J2CTNNk=&cssminify=yes' type='text/css' media='all' />
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/common.min.css */
</style>
<style id="wp-block-quote-inline-css">
.wp-block-quote{box-sizing:border-box;overflow-wrap:break-word}.wp-block-quote.is-large:where(:not(.is-style-plain)),.wp-block-quote.is-style-large:where(:not(.is-style-plain)){margin-bottom:1em;padding:0 1em}.wp-block-quote.is-large:where(:not(.is-style-plain)) p,.wp-block-quote.is-style-large:where(:not(.is-style-plain)) p{font-size:1.5em;font-style:italic;line-height:1.6}.wp-block-quote.is-large:where(:not(.is-style-plain)) cite,.wp-block-quote.is-large:where(:not(.is-style-plain)) footer,.wp-block-quote.is-style-large:where(:not(.is-style-plain)) cite,.wp-block-quote.is-style-large:where(:not(.is-style-plain)) footer{font-size:1.125em;text-align:right}.wp-block-quote>cite{display:block}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/quote/style.min.css */
</style>
<style id="wp-block-heading-inline-css">
h1:where(.wp-block-heading).has-background,h2:where(.wp-block-heading).has-background,h3:where(.wp-block-heading).has-background,h4:where(.wp-block-heading).has-background,h5:where(.wp-block-heading).has-background,h6:where(.wp-block-heading).has-background{padding:1.25em 2.375em}h1.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h1.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h2.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h2.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h3.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h3.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h4.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h4.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h5.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h5.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h6.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h6.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]){rotate:180deg}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/heading/style.min.css */
</style>
<style id="wp-block-image-inline-css">
.wp-block-image>a,.wp-block-image>figure>a{display:inline-block}.wp-block-image img{box-sizing:border-box;height:auto;max-width:100%;vertical-align:bottom}@media not (prefers-reduced-motion){.wp-block-image img.hide{visibility:hidden}.wp-block-image img.show{animation:show-content-image .4s}}.wp-block-image[style*=border-radius] img,.wp-block-image[style*=border-radius]>a{border-radius:inherit}.wp-block-image.has-custom-border img{box-sizing:border-box}.wp-block-image.aligncenter{text-align:center}.wp-block-image.alignfull>a,.wp-block-image.alignwide>a{width:100%}.wp-block-image.alignfull img,.wp-block-image.alignwide img{height:auto;width:100%}.wp-block-image .aligncenter,.wp-block-image .alignleft,.wp-block-image .alignright,.wp-block-image.aligncenter,.wp-block-image.alignleft,.wp-block-image.alignright{display:table}.wp-block-image .aligncenter>figcaption,.wp-block-image .alignleft>figcaption,.wp-block-image .alignright>figcaption,.wp-block-image.aligncenter>figcaption,.wp-block-image.alignleft>figcaption,.wp-block-image.alignright>figcaption{caption-side:bottom;display:table-caption}.wp-block-image .alignleft{float:left;margin:.5em 1em .5em 0}.wp-block-image .alignright{float:right;margin:.5em 0 .5em 1em}.wp-block-image .aligncenter{margin-left:auto;margin-right:auto}.wp-block-image :where(figcaption){margin-bottom:1em;margin-top:.5em}.wp-block-image.is-style-circle-mask img{border-radius:9999px}@supports ((-webkit-mask-image:none) or (mask-image:none)) or (-webkit-mask-image:none){.wp-block-image.is-style-circle-mask img{border-radius:0;-webkit-mask-image:url('data:image/svg+xml;utf8,<svg viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg"><circle cx="50" cy="50" r="50"/></svg>');mask-image:url('data:image/svg+xml;utf8,<svg viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg"><circle cx="50" cy="50" r="50"/></svg>');mask-mode:alpha;-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain}}:root :where(.wp-block-image.is-style-rounded img,.wp-block-image .is-style-rounded img){border-radius:9999px}.wp-block-image figure{margin:0}.wp-lightbox-container{display:flex;flex-direction:column;position:relative}.wp-lightbox-container img{cursor:zoom-in}.wp-lightbox-container img:hover+button{opacity:1}.wp-lightbox-container button{align-items:center;backdrop-filter:blur(16px) saturate(180%);background-color:rgba(90,90,90,.25);border:none;border-radius:4px;cursor:zoom-in;display:flex;height:20px;justify-content:center;opacity:0;padding:0;position:absolute;right:16px;text-align:center;top:16px;width:20px;z-index:100}@media not (prefers-reduced-motion){.wp-lightbox-container button{transition:opacity .2s ease}}.wp-lightbox-container button:focus-visible{outline:3px auto rgba(90,90,90,.25);outline:3px auto -webkit-focus-ring-color;outline-offset:3px}.wp-lightbox-container button:hover{cursor:pointer;opacity:1}.wp-lightbox-container button:focus{opacity:1}.wp-lightbox-container button:focus,.wp-lightbox-container button:hover,.wp-lightbox-container button:not(:hover):not(:active):not(.has-background){background-color:rgba(90,90,90,.25);border:none}.wp-lightbox-overlay{box-sizing:border-box;cursor:zoom-out;height:100vh;left:0;overflow:hidden;position:fixed;top:0;visibility:hidden;width:100%;z-index:100000}.wp-lightbox-overlay .wp-lightbox-close-button{align-items:center;cursor:pointer;display:flex;font-family:inherit;gap:8px;justify-content:center;line-height:1;min-height:40px;min-width:40px;padding:0 4px;position:absolute;right:calc(env(safe-area-inset-right) + 16px);top:calc(env(safe-area-inset-top) + 16px);z-index:5000000}.wp-lightbox-overlay .wp-lightbox-close-button:focus,.wp-lightbox-overlay .wp-lightbox-close-button:hover,.wp-lightbox-overlay .wp-lightbox-close-button:not(:hover):not(:active):not(.has-background){background:none;border:none}.wp-lightbox-overlay .wp-lightbox-close-button:has(.wp-lightbox-close-text:not([hidden])) .wp-lightbox-close-icon svg{height:1em;width:1em}.wp-lightbox-overlay .wp-lightbox-close-icon svg{display:block}.wp-lightbox-overlay .wp-lightbox-navigation-button-next,.wp-lightbox-overlay .wp-lightbox-navigation-button-prev{align-items:center;bottom:16px;cursor:pointer;display:flex;font-family:inherit;gap:4px;justify-content:center;line-height:1;min-height:40px;min-width:40px;padding:0 8px;position:absolute;z-index:2000002}.wp-lightbox-overlay .wp-lightbox-navigation-button-next[hidden],.wp-lightbox-overlay .wp-lightbox-navigation-button-prev[hidden]{display:none}@media (min-width:960px){.wp-lightbox-overlay .wp-lightbox-navigation-button-next,.wp-lightbox-overlay .wp-lightbox-navigation-button-prev{bottom:50%;transform:translateY(-50%)}}.wp-lightbox-overlay .wp-lightbox-navigation-button-next:focus,.wp-lightbox-overlay .wp-lightbox-navigation-button-next:hover,.wp-lightbox-overlay .wp-lightbox-navigation-button-next:not(:hover):not(:active):not(.has-background),.wp-lightbox-overlay .wp-lightbox-navigation-button-prev:focus,.wp-lightbox-overlay .wp-lightbox-navigation-button-prev:hover,.wp-lightbox-overlay .wp-lightbox-navigation-button-prev:not(:hover):not(:active):not(.has-background){background:none;border:none;padding:0 8px}.wp-lightbox-overlay .wp-lightbox-navigation-button-next:has(.wp-lightbox-navigation-text:not([hidden])) .wp-lightbox-navigation-icon svg,.wp-lightbox-overlay .wp-lightbox-navigation-button-prev:has(.wp-lightbox-navigation-text:not([hidden])) .wp-lightbox-navigation-icon svg{display:block;height:1.5em;width:1.5em}.wp-lightbox-overlay .wp-lightbox-navigation-button-prev{left:calc(env(safe-area-inset-left) + 16px)}.wp-lightbox-overlay .wp-lightbox-navigation-button-next{right:calc(env(safe-area-inset-right) + 16px)}.wp-lightbox-overlay .wp-lightbox-navigation-icon svg{vertical-align:middle}.wp-lightbox-overlay .lightbox-image-container{height:var(--wp--lightbox-container-height);left:50%;overflow:hidden;position:absolute;top:50%;transform:translate(-50%,-50%);transform-origin:top left;width:var(--wp--lightbox-container-width);z-index:2000001}.wp-lightbox-overlay .wp-block-image{align-items:center;box-sizing:border-box;display:flex;height:100%;justify-content:center;margin:0;position:relative;transform-origin:0 0;width:100%;z-index:3000000}.wp-lightbox-overlay .wp-block-image img{height:var(--wp--lightbox-image-height);min-height:var(--wp--lightbox-image-height);min-width:var(--wp--lightbox-image-width);width:var(--wp--lightbox-image-width)}.wp-lightbox-overlay .wp-block-image figcaption{display:none}.wp-lightbox-overlay button{background:none;border:none}.wp-lightbox-overlay .scrim{background-color:#fff;height:100%;opacity:.9;position:absolute;width:100%;z-index:2000000}.wp-lightbox-overlay.active{visibility:visible}@media not (prefers-reduced-motion){.wp-lightbox-overlay.active{animation:turn-on-visibility .25s both}.wp-lightbox-overlay.active img{animation:turn-on-visibility .35s both}.wp-lightbox-overlay.show-closing-animation:not(.active){animation:turn-off-visibility .35s both}.wp-lightbox-overlay.show-closing-animation:not(.active) img{animation:turn-off-visibility .25s both}.wp-lightbox-overlay.zoom.active{animation:none;opacity:1;visibility:visible}.wp-lightbox-overlay.zoom.active .lightbox-image-container{animation:lightbox-zoom-in .4s}.wp-lightbox-overlay.zoom.active .lightbox-image-container img{animation:none}.wp-lightbox-overlay.zoom.active .scrim{animation:turn-on-visibility .4s forwards}.wp-lightbox-overlay.zoom.show-closing-animation:not(.active){animation:none}.wp-lightbox-overlay.zoom.show-closing-animation:not(.active) .lightbox-image-container{animation:lightbox-zoom-out .4s}.wp-lightbox-overlay.zoom.show-closing-animation:not(.active) .lightbox-image-container img{animation:none}.wp-lightbox-overlay.zoom.show-closing-animation:not(.active) .scrim{animation:turn-off-visibility .4s forwards}}@keyframes show-content-image{0%{visibility:hidden}99%{visibility:hidden}to{visibility:visible}}@keyframes turn-on-visibility{0%{opacity:0}to{opacity:1}}@keyframes turn-off-visibility{0%{opacity:1;visibility:visible}99%{opacity:0;visibility:visible}to{opacity:0;visibility:hidden}}@keyframes lightbox-zoom-in{0%{transform:translate(calc((-100vw + var(--wp--lightbox-scrollbar-width))/2 + var(--wp--lightbox-initial-left-position)),calc(-50vh + var(--wp--lightbox-initial-top-position))) scale(var(--wp--lightbox-scale))}to{transform:translate(-50%,-50%) scale(1)}}@keyframes lightbox-zoom-out{0%{transform:translate(-50%,-50%) scale(1);visibility:visible}99%{visibility:visible}to{transform:translate(calc((-100vw + var(--wp--lightbox-scrollbar-width))/2 + var(--wp--lightbox-initial-left-position)),calc(-50vh + var(--wp--lightbox-initial-top-position))) scale(var(--wp--lightbox-scale));visibility:hidden}}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/image/style.min.css */
</style>
<style id="wp-block-list-inline-css">
ol,ul{box-sizing:border-box}:root :where(.wp-block-list.has-background){padding:1.25em 2.375em}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/list/style.min.css */
</style>
<style id="wp-block-paragraph-inline-css">
.is-small-text{font-size:.875em}.is-regular-text{font-size:1em}.is-large-text{font-size:2.25em}.is-larger-text{font-size:3em}.has-drop-cap:not(:focus):first-letter{float:left;font-size:8.4em;font-style:normal;font-weight:100;line-height:.68;margin:.05em .1em 0 0;text-transform:uppercase}body.rtl .has-drop-cap:not(:focus):first-letter{float:none;margin-left:.1em}p.has-drop-cap.has-background{overflow:hidden}:root :where(p.has-background){padding:1.25em 2.375em}:where(p.has-text-color:not(.has-link-color)) a{color:inherit}p.has-text-align-left[style*="writing-mode:vertical-lr"],p.has-text-align-right[style*="writing-mode:vertical-rl"]{rotate:180deg}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/paragraph/style.min.css */
</style>
<style id="wp-block-library-inline-css-extra">
.has-text-align-justify {
text-align:justify;
}
.has-text-align-justify{text-align:justify;}
/*# sourceURL=wp-block-library-inline-css */
</style>
<style id="classic-theme-styles-inline-css">
.wp-block-button__link{background-color:#32373c;border-radius:9999px;box-shadow:none;color:#fff;font-size:1.125em;padding:calc(.667em + 2px) calc(1.333em + 2px);text-decoration:none}.wp-block-file__button{background:#32373c;color:#fff}.wp-block-accordion-heading{margin:0}.wp-block-accordion-heading__toggle{background-color:inherit!important;color:inherit!important}.wp-block-accordion-heading__toggle:not(:focus-visible){outline:none}.wp-block-accordion-heading__toggle:focus,.wp-block-accordion-heading__toggle:hover{background-color:inherit!important;border:none;box-shadow:none;color:inherit;padding:var(--wp--preset--spacing--20,1em) 0;text-decoration:none}.wp-block-accordion-heading__toggle:focus-visible{outline:auto;outline-offset:0}.wp-block-tab:not(.has-text-color){color:inherit!important}.wp-block-tab:not(.has-background){background-color:inherit!important}.wp-block-tab:focus,.wp-block-tab:hover{text-decoration:none}.wp-block-tab:focus-visible{outline:auto;outline-offset:0}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/classic.min.css */
</style>
<style id="coblocks-inline-format-inline-css">
.uppercase{text-transform:uppercase}
/*# sourceURL=coblocks-inline-format-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-12-1' href='/_static/??-eJx9jEEOAiEQBD/kQECjeDC+ZWEnijIwYWA3/l72op689KFTVXplCCU3zE1z6reYRYfiUwlP0VYZpwxIJE4IFRd10HOU9iFA2iuhCiI7/ROiDt9WxfETT20jCOc4YUIa2D9t5eGA91xRBMZS7ATtPkTZvCtdzNEZt7f2fHq8Afl7R7w=&cssminify=yes' type='text/css' media='all' />
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #fff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--color--dark-green: #1c7c7c;--wp--preset--color--dark-gray: #666;--wp--preset--color--medium-gray: #999;--wp--preset--color--light-gray: #eee;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--font-family--albert-sans: 'Albert Sans', sans-serif;--wp--preset--font-family--alegreya: Alegreya, serif;--wp--preset--font-family--arvo: Arvo, serif;--wp--preset--font-family--bodoni-moda: 'Bodoni Moda', serif;--wp--preset--font-family--bricolage-grotesque: 'Bricolage Grotesque', sans-serif;--wp--preset--font-family--cabin: Cabin, sans-serif;--wp--preset--font-family--chivo: Chivo, sans-serif;--wp--preset--font-family--commissioner: Commissioner, sans-serif;--wp--preset--font-family--cormorant: Cormorant, serif;--wp--preset--font-family--courier-prime: 'Courier Prime', monospace;--wp--preset--font-family--crimson-pro: 'Crimson Pro', serif;--wp--preset--font-family--dm-mono: 'DM Mono', monospace;--wp--preset--font-family--dm-sans: 'DM Sans', sans-serif;--wp--preset--font-family--dm-serif-display: 'DM Serif Display', serif;--wp--preset--font-family--domine: Domine, serif;--wp--preset--font-family--eb-garamond: 'EB Garamond', serif;--wp--preset--font-family--epilogue: Epilogue, sans-serif;--wp--preset--font-family--fahkwang: Fahkwang, sans-serif;--wp--preset--font-family--figtree: Figtree, sans-serif;--wp--preset--font-family--fira-sans: 'Fira Sans', sans-serif;--wp--preset--font-family--fjalla-one: 'Fjalla One', sans-serif;--wp--preset--font-family--fraunces: Fraunces, serif;--wp--preset--font-family--gabarito: Gabarito, system-ui;--wp--preset--font-family--ibm-plex-mono: 'IBM Plex Mono', monospace;--wp--preset--font-family--ibm-plex-sans: 'IBM Plex Sans', sans-serif;--wp--preset--font-family--ibarra-real-nova: 'Ibarra Real Nova', serif;--wp--preset--font-family--instrument-serif: 'Instrument Serif', serif;--wp--preset--font-family--inter: Inter, sans-serif;--wp--preset--font-family--josefin-sans: 'Josefin Sans', sans-serif;--wp--preset--font-family--jost: Jost, sans-serif;--wp--preset--font-family--libre-baskerville: 'Libre Baskerville', serif;--wp--preset--font-family--libre-franklin: 'Libre Franklin', sans-serif;--wp--preset--font-family--literata: Literata, serif;--wp--preset--font-family--lora: Lora, serif;--wp--preset--font-family--merriweather: Merriweather, serif;--wp--preset--font-family--montserrat: Montserrat, sans-serif;--wp--preset--font-family--newsreader: Newsreader, serif;--wp--preset--font-family--noto-sans-mono: 'Noto Sans Mono', sans-serif;--wp--preset--font-family--nunito: Nunito, sans-serif;--wp--preset--font-family--open-sans: 'Open Sans', sans-serif;--wp--preset--font-family--overpass: Overpass, sans-serif;--wp--preset--font-family--pt-serif: 'PT Serif', serif;--wp--preset--font-family--petrona: Petrona, serif;--wp--preset--font-family--piazzolla: Piazzolla, serif;--wp--preset--font-family--playfair-display: 'Playfair Display', serif;--wp--preset--font-family--plus-jakarta-sans: 'Plus Jakarta Sans', sans-serif;--wp--preset--font-family--poppins: Poppins, sans-serif;--wp--preset--font-family--raleway: Raleway, sans-serif;--wp--preset--font-family--roboto: Roboto, sans-serif;--wp--preset--font-family--roboto-slab: 'Roboto Slab', serif;--wp--preset--font-family--rubik: Rubik, sans-serif;--wp--preset--font-family--rufina: Rufina, serif;--wp--preset--font-family--sora: Sora, sans-serif;--wp--preset--font-family--source-sans-3: 'Source Sans 3', sans-serif;--wp--preset--font-family--source-serif-4: 'Source Serif 4', serif;--wp--preset--font-family--space-mono: 'Space Mono', monospace;--wp--preset--font-family--syne: Syne, sans-serif;--wp--preset--font-family--texturina: Texturina, serif;--wp--preset--font-family--urbanist: Urbanist, sans-serif;--wp--preset--font-family--work-sans: 'Work Sans', sans-serif;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-color{color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-color{color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-color{color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-color{color: var(--wp--preset--color--light-gray) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-background-color{background-color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-background-color{background-color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-background-color{background-color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-background-color{background-color: var(--wp--preset--color--light-gray) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-border-color{border-color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-border-color{border-color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-border-color{border-color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-border-color{border-color: var(--wp--preset--color--light-gray) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}.has-albert-sans-font-family{font-family: var(--wp--preset--font-family--albert-sans) !important;}.has-alegreya-font-family{font-family: var(--wp--preset--font-family--alegreya) !important;}.has-arvo-font-family{font-family: var(--wp--preset--font-family--arvo) !important;}.has-bodoni-moda-font-family{font-family: var(--wp--preset--font-family--bodoni-moda) !important;}.has-bricolage-grotesque-font-family{font-family: var(--wp--preset--font-family--bricolage-grotesque) !important;}.has-cabin-font-family{font-family: var(--wp--preset--font-family--cabin) !important;}.has-chivo-font-family{font-family: var(--wp--preset--font-family--chivo) !important;}.has-commissioner-font-family{font-family: var(--wp--preset--font-family--commissioner) !important;}.has-cormorant-font-family{font-family: var(--wp--preset--font-family--cormorant) !important;}.has-courier-prime-font-family{font-family: var(--wp--preset--font-family--courier-prime) !important;}.has-crimson-pro-font-family{font-family: var(--wp--preset--font-family--crimson-pro) !important;}.has-dm-mono-font-family{font-family: var(--wp--preset--font-family--dm-mono) !important;}.has-dm-sans-font-family{font-family: var(--wp--preset--font-family--dm-sans) !important;}.has-dm-serif-display-font-family{font-family: var(--wp--preset--font-family--dm-serif-display) !important;}.has-domine-font-family{font-family: var(--wp--preset--font-family--domine) !important;}.has-eb-garamond-font-family{font-family: var(--wp--preset--font-family--eb-garamond) !important;}.has-epilogue-font-family{font-family: var(--wp--preset--font-family--epilogue) !important;}.has-fahkwang-font-family{font-family: var(--wp--preset--font-family--fahkwang) !important;}.has-figtree-font-family{font-family: var(--wp--preset--font-family--figtree) !important;}.has-fira-sans-font-family{font-family: var(--wp--preset--font-family--fira-sans) !important;}.has-fjalla-one-font-family{font-family: var(--wp--preset--font-family--fjalla-one) !important;}.has-fraunces-font-family{font-family: var(--wp--preset--font-family--fraunces) !important;}.has-gabarito-font-family{font-family: var(--wp--preset--font-family--gabarito) !important;}.has-ibm-plex-mono-font-family{font-family: var(--wp--preset--font-family--ibm-plex-mono) !important;}.has-ibm-plex-sans-font-family{font-family: var(--wp--preset--font-family--ibm-plex-sans) !important;}.has-ibarra-real-nova-font-family{font-family: var(--wp--preset--font-family--ibarra-real-nova) !important;}.has-instrument-serif-font-family{font-family: var(--wp--preset--font-family--instrument-serif) !important;}.has-inter-font-family{font-family: var(--wp--preset--font-family--inter) !important;}.has-josefin-sans-font-family{font-family: var(--wp--preset--font-family--josefin-sans) !important;}.has-jost-font-family{font-family: var(--wp--preset--font-family--jost) !important;}.has-libre-baskerville-font-family{font-family: var(--wp--preset--font-family--libre-baskerville) !important;}.has-libre-franklin-font-family{font-family: var(--wp--preset--font-family--libre-franklin) !important;}.has-literata-font-family{font-family: var(--wp--preset--font-family--literata) !important;}.has-lora-font-family{font-family: var(--wp--preset--font-family--lora) !important;}.has-merriweather-font-family{font-family: var(--wp--preset--font-family--merriweather) !important;}.has-montserrat-font-family{font-family: var(--wp--preset--font-family--montserrat) !important;}.has-newsreader-font-family{font-family: var(--wp--preset--font-family--newsreader) !important;}.has-noto-sans-mono-font-family{font-family: var(--wp--preset--font-family--noto-sans-mono) !important;}.has-nunito-font-family{font-family: var(--wp--preset--font-family--nunito) !important;}.has-open-sans-font-family{font-family: var(--wp--preset--font-family--open-sans) !important;}.has-overpass-font-family{font-family: var(--wp--preset--font-family--overpass) !important;}.has-pt-serif-font-family{font-family: var(--wp--preset--font-family--pt-serif) !important;}.has-petrona-font-family{font-family: var(--wp--preset--font-family--petrona) !important;}.has-piazzolla-font-family{font-family: var(--wp--preset--font-family--piazzolla) !important;}.has-playfair-display-font-family{font-family: var(--wp--preset--font-family--playfair-display) !important;}.has-plus-jakarta-sans-font-family{font-family: var(--wp--preset--font-family--plus-jakarta-sans) !important;}.has-poppins-font-family{font-family: var(--wp--preset--font-family--poppins) !important;}.has-raleway-font-family{font-family: var(--wp--preset--font-family--raleway) !important;}.has-roboto-font-family{font-family: var(--wp--preset--font-family--roboto) !important;}.has-roboto-slab-font-family{font-family: var(--wp--preset--font-family--roboto-slab) !important;}.has-rubik-font-family{font-family: var(--wp--preset--font-family--rubik) !important;}.has-rufina-font-family{font-family: var(--wp--preset--font-family--rufina) !important;}.has-sora-font-family{font-family: var(--wp--preset--font-family--sora) !important;}.has-source-sans-3-font-family{font-family: var(--wp--preset--font-family--source-sans-3) !important;}.has-source-serif-4-font-family{font-family: var(--wp--preset--font-family--source-serif-4) !important;}.has-space-mono-font-family{font-family: var(--wp--preset--font-family--space-mono) !important;}.has-syne-font-family{font-family: var(--wp--preset--font-family--syne) !important;}.has-texturina-font-family{font-family: var(--wp--preset--font-family--texturina) !important;}.has-urbanist-font-family{font-family: var(--wp--preset--font-family--urbanist) !important;}.has-work-sans-font-family{font-family: var(--wp--preset--font-family--work-sans) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-14-1' href='/_static/??/wp-content/themes/pub/penscratch-2/css/reset.css,/wp-content/themes/pub/penscratch-2/style.css?m=1779214100j&cssminify=yes' type='text/css' media='all' />
<style id="penscratch-2-style-inline-css">
.cat-links, .byline .author, .group-blog .byline + .sep, .single .byline + .sep { clip: rect(1px, 1px, 1px, 1px); height: 1px; position: absolute; overflow: hidden; width: 1px; }
/*# sourceURL=penscratch-2-style-inline-css */
</style>
<link rel='stylesheet' id='penscratch-2-fonts-css' href='https://fonts-api.wp.com/css?family=Roboto+Slab%3A300%2C400%2C700&#038;subset=latin%2Clatin-ext' media='all' />
<link crossorigin='anonymous' rel='stylesheet' id='all-css-16-1' href='/_static/??-eJyNjsEKAjEMRH/ImO0iLB7Eb+mG2K22aWmyLP691YuIIN7mHd7M4FaBihiLoS2cWbGuM1YWpeaNFhiRVHFOhW6673GHfyhRCNXuiWGrVPKXl1eoaQ1RFBv37tBjeA298ZcUuEB/5C0W+QC4JB/bUz3nk5um4+gObhiuD4S6VWY=&cssminify=yes' type='text/css' media='all' />
<link crossorigin='anonymous' rel='stylesheet' id='print-css-17-1' href='/wp-content/mu-plugins/global-print/global-print.css?m=1465851035i&cssminify=yes' type='text/css' media='print' />
<style id="jetpack-global-styles-frontend-style-inline-css">
:root { --font-headings: unset; --font-base: unset; --font-headings-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif; --font-base-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;}
/*# sourceURL=jetpack-global-styles-frontend-style-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-20-1' href='/_static/??-eJyNjcEKAjEMRH/ImhVlqwfxU6TbhrZrmhTTIv69u+JFvHgZ5sHwBh7VeOGG3KB0U6nHzAoztur87cNQRNYInVBBk7tjcCE83zVz3HrVDfxvumb2MPVMAVR8dmRIougX/DhbwrK8pwNEksnROriU884e9+PJDnaYXxwFS20=&cssminify=yes' type='text/css' media='all' />
<script id="jetpack_related-posts-js-extra">
var related_posts_js_options = {"post_heading":"h4"};
//# sourceURL=jetpack_related-posts-js-extra
</script>
<script id="wpcom-actionbar-placeholder-js-extra">
var actionbardata = {"siteID":"23592848","postID":"8869","siteURL":"https://blog.acolyer.org","xhrURL":"https://blog.acolyer.org/wp-admin/admin-ajax.php","nonce":"e8f3aebd24","isLoggedIn":"","statusMessage":"","subsEmailDefault":"instantly","proxyScriptUrl":"https://s0.wp.com/wp-content/js/wpcom-proxy-request.js?m=1513050504i&amp;ver=20211021","shortlink":"https://wp.me/p1AZzO-2j3","i18n":{"followedText":"New posts from this site will now appear in your \u003Ca href=\"https://wordpress.com/reader\"\u003EReader\u003C/a\u003E","foldBar":"Collapse this bar","unfoldBar":"Expand this bar","shortLinkCopied":"Shortlink copied to clipboard."}};
//# sourceURL=wpcom-actionbar-placeholder-js-extra
</script>
<script id="jetpack-mu-wpcom-settings-js-before">
var JETPACK_MU_WPCOM_SETTINGS = {"assetsUrl":"https://s1.wp.com/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/moon/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/"};
//# sourceURL=jetpack-mu-wpcom-settings-js-before
</script>
<script crossorigin='anonymous' type='text/javascript' src='/_static/??/wp-content/mu-plugins/jetpack-plugin/moon/_inc/build/related-posts/related-posts.min.js,/wp-content/js/rlt-proxy.js?m=1788798821j'></script>
<script id="rlt-proxy-js-after">
rltInitialize( {"token":null,"iframeOrigins":["https:\/\/widgets.wp.com"]} );
//# sourceURL=rlt-proxy-js-after
</script>
<link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://adriancolyer.wordpress.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress.com" />
<link rel="canonical" href="https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/" />
<link rel='shortlink' href='https://wp.me/p1AZzO-2j3' />
<link rel="alternate" type="application/json+oembed" href="https://public-api.wordpress.com/oembed/?format=json&amp;url=https%3A%2F%2Fblog.acolyer.org%2F2019%2F07%2F05%2Fautomating-chaos-experiments-in-production%2F&amp;for=wpcom-auto-discovery" /><link rel="alternate" type="application/xml+oembed" href="https://public-api.wordpress.com/oembed/?format=xml&amp;url=https%3A%2F%2Fblog.acolyer.org%2F2019%2F07%2F05%2Fautomating-chaos-experiments-in-production%2F&amp;for=wpcom-auto-discovery" />
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="Automating chaos experiments in production" />
<meta property="og:url" content="https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/" />
<meta property="og:description" content="Automating chaos experiments in production Basiri et al., ICSE 2019 Are you ready to take your system assurance programme to the next level? This is a fascinating paper from members of Netflix’s Re…" />
<meta property="article:published_time" content="2019-07-05T05:00:00+00:00" />
<meta property="article:modified_time" content="2019-06-28T13:05:51+00:00" />
<meta property="og:site_name" content="the morning paper" />
<meta property="og:image" content="https://blog.acolyer.org/wp-content/uploads/2019/06/chaoseng-fig-7.jpeg?w=640" />
<meta property="og:image:alt" content="" />
<meta property="og:locale" content="en_US" />
<meta property="article:publisher" content="https://www.facebook.com/WordPresscom" />
<meta name="twitter:text:title" content="Automating chaos experiments in&nbsp;production" />
<meta name="twitter:image" content="https://blog.acolyer.org/wp-content/uploads/2019/06/chaoseng-fig-7.jpeg?w=144" />
<meta name="twitter:card" content="summary" />
<!-- End Jetpack Open Graph Tags -->
<link rel="shortcut icon" type="image/x-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=32" sizes="16x16" />
<link rel="icon" type="image/x-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=32" sizes="16x16" />
<link rel="apple-touch-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=114" />
<link rel='openid.server' href='https://blog.acolyer.org/?openidserver=1' />
<link rel='openid.delegate' href='https://blog.acolyer.org/' />
<link rel="search" type="application/opensearchdescription+xml" href="https://blog.acolyer.org/osd.xml" title="the morning paper" />
<link rel="search" type="application/opensearchdescription+xml" href="https://s1.wp.com/opensearch.xml" title="WordPress.com" />
<meta name="theme-color" content="#ffffff" />
<meta name="description" content="Automating chaos experiments in production Basiri et al., ICSE 2019 Are you ready to take your system assurance programme to the next level? This is a fascinating paper from members of Netflix’s Resilience Engineering team describing their chaos engineering initiatives: automated controlled experiments designed to verify hypotheses about how the system should behave under gray&hellip;" />
<style id="custom-background-css">
body.custom-background { background-color: #ffffff; }
</style>
<style type="text/css" id="custom-colors-css"> .widget a:hover,
.site-footer .widget a:hover {
color: #666;
}
.jetpack-social-navigation ul a:hover { color: #FFFFFF;}
body { background-color: #ffffff;}
.jetpack-social-navigation ul a:hover { background-color: #1c7c7c;}
.jetpack-recipe-title { border-color: #1c7c7c;}
.toggled .menu-toggle, .wp_widget_tag_cloud a:hover, .tags-links a:hover { border-color: #1B7878;}
.button:hover, .button:focus, .button:active, button:hover, button:focus, button:active, input[type="button"]:hover, input[type="button"]:focus, input[type="button"]:active, input[type="reset"]:hover, input[type="reset"]:focus, input[type="reset"]:active, input[type="submit"]:hover, input[type="submit"]:focus, input[type="submit"]:active, #infinite-handle span:hover { border-color: #1B7878;}
.button:hover, .button:focus, .button:active, button:hover, button:focus, button:active, input[type="button"]:hover, input[type="button"]:focus, input[type="button"]:active, input[type="reset"]:hover, input[type="reset"]:focus, input[type="reset"]:active, input[type="submit"]:hover, input[type="submit"]:focus, input[type="submit"]:active, #infinite-handle span:hover { color: #1B7878;}
.menu-toggle:hover #menu-icon, .toggled #menu-icon { fill: #1B7878;}
a, a:visited, .site-footer a:hover, .main-navigation li:hover > a, .current_page_item a, .current-menu-item a, .toggled .menu-toggle, .entry-footer a:hover, .entry-meta a:hover, .entry-title a:hover, .site-main .post-navigation .meta-nav, .site-main .paging-navigation .meta-nav, .comment-meta .comment-metadata a:hover, .required, .widget a, #wp-calendar a, .wp_widget_tag_cloud a:hover, #infinite-footer .container a:hover, #infinite-handle span, .tags-links a:hover, .current_page_item a, .current-menu-item a, .current_page_item a:visited, .current-menu-item a:visited { color: #1B7878;}
</style>
<link rel="stylesheet" id="custom-css-css" type="text/css" href="https://s2.wp.com/?custom-css=1&#038;csblog=1AZzO&#038;cscache=6&#038;csrev=16" />
</head>
<body class="wp-singular post-template-default single single-post postid-8869 single-format-standard custom-background wp-embed-responsive wp-theme-pubpenscratch-2 customizer-styles-applied no-sidebar jetpack-reblog-enabled categories-hidden author-hidden custom-colors">
<div id="page" class="hfeed site">
<a class="skip-link screen-reader-text" href="#content">Skip to content</a>
<header id="masthead" class="site-header" role="banner">
<div class="site-branding">
<p class="site-title"><a href="https://blog.acolyer.org/" rel="home">the morning paper</a></p>
<p class="site-description">a random walk through Computer Science research, by Adrian Colyer</p>
</div>
<nav id="site-navigation" class="main-navigation" role="navigation">
<button class="menu-toggle">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><rect x="0" fill="none" width="16" height="16"/><g><path id="menu-icon" d="M0 14h16v-2H0v2zM0 2v2h16V2H0zm0 7h16V7H0v2z"/></g></svg>
Menu </button>
<div class="menu"><ul>
<li class="page_item page-item-2"><a href="https://blog.acolyer.org/about/">About</a></li>
<li class="page_item page-item-8716"><a href="https://blog.acolyer.org/archives/">Archives</a></li>
<li class="page_item page-item-5027"><a href="https://blog.acolyer.org/infoq-quarterly-review-editions/">InfoQ QR Editions</a></li>
<li class="page_item page-item-8713"><a href="https://blog.acolyer.org/search/">Search</a></li>
<li class="page_item page-item-406"><a href="https://blog.acolyer.org/email-subs/">Subscribe</a></li>
<li class="page_item page-item-8719"><a href="https://blog.acolyer.org/tags/">Tags</a></li>
<li class="page_item page-item-6991"><a href="https://blog.acolyer.org/privacy/">Privacy</a></li>
</ul></div>
</nav><!-- #site-navigation -->
</header><!-- #masthead -->
<div id="content" class="site-content">
<div id="primary" class="content-area">
<main id="main" class="site-main" role="main">
<article id="post-8869" class="post-8869 post type-post status-publish format-standard hentry category-uncategorized tag-operations tag-software-engineering">
<header class="entry-header">
<h1 class="entry-title">Automating chaos experiments in&nbsp;production</h1>
<div class="entry-meta">
<span class="posted-on"><a href="https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/" rel="bookmark"><time class="entry-date published" datetime="2019-07-05T06:00:00+01:00">July 5, 2019</time><time class="updated" datetime="2019-06-28T14:05:51+01:00">June 28, 2019</time></a></span><span class="byline"><span class="author vcard"><span class="sep"> ~ </span><a class="url fn n" href="https://blog.acolyer.org/author/adriancolyer/">adriancolyer</a></span></span> </div><!-- .entry-meta -->
</header><!-- .entry-header -->
<div class="entry-content">
<p class="wp-block-paragraph"><a href="https://arxiv.org/abs/1905.04648">Automating chaos experiments in production</a> Basiri et al., <em>ICSE 2019</em></p>
<p class="wp-block-paragraph">Are you ready to take your system assurance programme to the next level? This is a fascinating paper from members of Netflix’s Resilience Engineering team describing their chaos engineering initiatives: automated controlled experiments designed to verify hypotheses about how the system should behave under <a href="https://blog.acolyer.org/2017/06/15/gray-failure-the-achilles-heel-of-cloud-scale-systems/">gray failure</a> conditions, and to probe for and flush out any weaknesses. The ‘controlled’ part is important here because given the scale and complexity of the environment under test, the only meaningful place to do this is in production with real users.</p>
<p class="wp-block-paragraph">Maybe that sounds scary, but one of the interesting perspectives this paper brings is to make you realise that it’s really not so different from any other change you might be rolling out into production (e.g. a bug fix, configuration change, new feature, or A/B test). In all cases we need to be able to carefully monitor the impact on the system, and back out if things start going badly wrong. Moreover, just like an A/B test, we’ll be collecting metrics while the experiment is underway and performing statistical analysis at the end to interpret the results.</p>
<p class="wp-block-paragraph">Netflix’s system is deployed on the public cloud as complex set of interacting microservices.</p>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
In this type of environment, there are many potential sources of failure, stemming from the infrastructure itself (e.g. degraded hardware, transient networking problem) or, more often, because of some change deployed by Netflix engineers that did not have the intended effect.
</p></blockquote>
<p class="wp-block-paragraph">They use a combination of timeouts, retries, and fallbacks to try to mitigate the effects of these failures, but these don’t get exercised as often as the happy path, so how can we be confident they’ll work as intended when called upon? (And on top of that, there are all of the hard-to-predict emergent behaviours arising from interactions between multiple components). Netflix have a ChAP whose role it is to help provide that confidence: a <strong>Ch</strong>aos <strong>A</strong>utomation <strong>P</strong>latform.</p>
<h3 class="wp-block-heading">The Chaos automation platform</h3>
<p class="wp-block-paragraph">ChAP runs chaos engineering experiments within Netflix’s production systems.</p>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
Most ChAP experiments focus on evaluating whether the overall system would remain healthy if one of the services degraded. Two failure modes we focus on are a service becoming slower (increase in response latency) or a service failing outright (returning errors).
</p></blockquote>
<p class="wp-block-paragraph">ChAP works with a number of other Netflix systems to carry out its tasks: <a href="https://medium.com/netflix-techblog/fit-failure-injection-testing-35d8e2a9bb2">FIT</a> for fault injection, <a href="https://www.spinnaker.io/">Spinnaker</a> for provisioning baseline and canary clusters, <a href="https://github.com/Netflix/zuul">Zuul</a> to manage user traffic routing to the canary or baseline, <a href="https://medium.com/netflix-techblog/introducing-atlas-netflixs-primary-telemetry-platform-bd31f4d8ed9a">Atlas</a> and <a href="https://medium.com/netflix-techblog/stream-processing-with-mantis-78af913f51a6">Mantis</a> for telemetry during the experiment, <a href="https://medium.com/netflix-techblog/lumen-custom-self-service-dashboarding-for-netflix-8c56b541548c">Lumen</a> dashboards for displaying metrics data, and the <a href="https://medium.com/netflix-techblog/automated-canary-analysis-at-netflix-with-kayenta-3260bc7acc69">Kayenta</a> automated canary analysis system.</p>
<p class="wp-block-paragraph">An example hypotheses to be tested is that users should still be able to stream video even if the bookmarking service (which remembers their last watched location in a video) is down. To test this a Netflix engineer can create an experiment using the ChAP UI, &#8220;<em>fail RPCs to the bookmark service and verify the system in general and the API service in particular remain healthy</em>.&#8221;</p>
<p class="wp-block-paragraph">Typically the baseline and canary clusters deployed for an experiment will be 1% of the size of the original cluster. Users selected (by Zuul) to be part of the canary group will be routed to the canary cluster with an additional fault-injection rule to fail calls to the bookmark service.</p>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
There are two main classes of metric we use to determine whether the injected fault has had a negative impact on system health: (i) key performance indicators (KPIs) for the users in the canary group versus the baseline group; (ii) health metrics for the canary cluster versus the baseline cluster.
</p></blockquote>
<p class="wp-block-paragraph">After the experiment ends, Kayenta performs a statistical analysis of metrics collected from the canary cluster and compares them to those from the baseline cluster, to determine if there has been a statistically significant impact on any metrics of interest.</p>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
While Kayenta was originally designed for evaluating the health of canary deployments, we found it to be an excellent match for analyzing the results of ChAP experiments.
</p></blockquote>
<h3 class="wp-block-heading">Safeguards</h3>
<p class="wp-block-paragraph">ChAP has a number of safeguards in place to minimise the blast radius of experiments:</p>
<ul class="wp-block-list"><li>Experiments are only run during business hours on weekdays</li><li>Experiments are not run when a region is failed-over for independent reasons</li><li>The set of all concurrently executing ChAP experiments (yes, there can be lot of them executing pretty much continuously, which is where the dedicated baseline and canary clusters come in) cannot impact more than 5% of the total traffic in any one geographical region.</li><li>If ChAP detects excessive customer impact during an experiment, the experiment is stopped immediately </li></ul>
<h3 class="wp-block-heading">Defining and running experiments</h3>
<p class="wp-block-paragraph">Initially ChAP was rolled out with a self-serve model whereby teams could define and run their own experiments. This mode is still available but take up from busy service teams was limited. So the Resilience Engineering team developed <em>Monocle</em>, a service for automatically uncovering service dependencies, and using this information to automatically generate and execute experiments.</p>
<p class="wp-block-paragraph">The early failure injection testing mechanisms from Chaos Monkey and friends were like acts of random vandalism. Monocle is more of an intelligent probing, seeking out any weakness a service may have.</p>
<p class="wp-block-paragraph">The dependency analysis part of Monocle is pretty cool all by itself, even before we get to the experiment generation. Monocle grabs data from the Atlas telemetry system, the tracing system, and by querying runtime servers for configuration information such as time values. RPCs at Netflix are wrapped as Hystrix commands. For each such command, Monocle shows the following information:</p>
<ul class="wp-block-list"><li>Percentage of inbound requests triggering an invocation of the command</li><li>Whether it is believed to be safe to fail</li><li>Whether it is configured with a fallback</li><li>The configured timeouts</li><li>The observed latencies over the last 2 weeks (P90, P99, P99.5)</li><li>The thread pool size</li><li>The observed number of active threads over the past two weeks</li><li>Which RPC clients the command wraps, if any</li><li>Any known impacts associated with the command</li></ul>
<p class="wp-block-paragraph">For every RPC client Monocle shows:</p>
<ul class="wp-block-list"><li>Timeout and retry configurations</li><li>Percentage of inbound requests triggering an RPC call</li><li>Maximum observed invocation rate in the past 2 weeks</li><li>Which Hystrix commands wrap it, if any, and if they are safe to fail</li><li>Any known impacts associated with the RPC client or the commands that wrap it.</li></ul>
<figure class="wp-block-image"><img src="https://blog.acolyer.org/wp-content/uploads/2019/06/chaoseng-fig-7.jpeg?w=640" alt="" /></figure>
<p class="wp-block-paragraph"><br>
(<a href="https://blog.acolyer.org/wp-content/uploads/2019/06/chaoseng-fig-7.jpeg">Enlarge</a>)</p>
<p class="wp-block-paragraph">In the dashboard screenshot above you can see yellow and red attention icons highlighting warnings and vulnerabilities. These have tooltips with more detail, for example:</p>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
<strong>Warning</strong>: Hystrix Command Timeout is Misaligned with RPC Client.</p><p> Timeout (1000 ms) is less than the max computed timeout of the wrapped RPC client (4000 ms). This means that Hystrix will give up waiting on RPC&#8230;
</p></blockquote>
<p class="wp-block-paragraph">Using all this information, Monocle generates resilience experiments automatically for RPC client and Hystrix command dependencies. For each dependency there are three basic experiment types:</p>
<ol class="wp-block-list"><li>Failure</li><li>Latency just below the configured timeout (highest timeout &#8211; P99 latency over the past 7 days + 5% buffer)</li><li>Latency causing failure (highest configured timeout + 5% buffer)</li></ol>
<p class="wp-block-paragraph">As you can imagine, with lots of services and large fan-outs, that’s a <em>lot</em> of potential experiments that could be run.</p>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
Monocle using heuristics to try to identify the experiments with the highest likelihood of finding a vulnerability.
</p></blockquote>
<p class="wp-block-paragraph">Each dependency is assigned a <em>criticality score</em> which is a product of the dependency priority (Hystrix commands get priority over RPC clients), how frequently the dependency is triggered compared to all other dependencies in the cluster, the retry factor, and the number of dependent clients.</p>
<p class="wp-block-paragraph">The criticality score is combined with a safety score and experiment weight (failure experiments, then latency, than failure inducing latency) to produce the final prioritization score. The prioritized list is filtered to remove unsafe experiments, experiments that are already running, experiments in a failed state (previously failed, and not yet inspected by a user), and experiments that have run recently. After that Monocle gets to work and starts running the remaining experiments in priority order.</p>
<p class="wp-block-paragraph">It would be great to have more detail on the quantitative or qualitative improvements to resiliency at Netflix that Monocle and chaos engineering have made, but for the moment it seems we will have to suffice ourselves with this single sentence:</p>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
To date, experiments generated by Monocle have revealed several cases where timeouts were set incorrectly and fallbacks revealed the service to be more business critical than the owner had intended.
</p></blockquote>
<p class="wp-block-paragraph">When an automated experiment reveals a potential vulnerability, a member of the Resilience Engineering team spends time manually analysing the results of the experiment before flagging them to the owner. This reduces the false positive rate and helps build trust in the system, but also sounds like the current bottleneck in the process (&#8220;<em>this was generally a time-consuming and tedious process</em>&#8220;).</p>
<h3 class="wp-block-heading">On error rates</h3>
<p class="wp-block-paragraph">If you’ve read the <a href="https://landing.google.com/sre/books/">SRE book</a> you’ve probably come across the &#8220;four golden signals&#8221; (p60): latency, throughput, error rate, and saturation. So it’s interesting to read here that&#8230;</p>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
Although intuitively error rates seem like a reliable signal for identifying problems with the system, we found them to have some undesirable properties for use as an experimental measure.
</p></blockquote>
<p class="wp-block-paragraph">The issue seems to stem from the fact that a single device may error over and over again, bumping up the error count, and hence ultimately the error rate. So if an error-prone device happens to be assigned to a canary cluster it can generate a spurious error signal from the perspective of the experiment. Due to this the threshold for the automatic stop safety mechanism had be substantially increased. (Tracking error rates per device might help with this??).</p>
<h3 class="wp-block-heading">There are many types of controlled experiment</h3>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
While the original goal for ChAP was to run fault injection experiments, we have discovered that the platform itself can be used for other types of experiments. We are currently extending ChAP to support load testing experiments, similar in spirit to <a href="https://engineering.linkedin.com/blog/2017/02/redliner--how-linkedin-determines-the-capacity-limits-of-its-ser">RedLiner</a>. In addition, some self-serve users have begun experimenting with using ChAP for canary deployments because of the additional analysis that ChAP provides.
</p></blockquote>
<p class="wp-block-paragraph">As a follow-up to this paper you might also enjoy Nora Jones’ article on ‘<a href="https://medium.com/@njones_18523/chaos-engineering-traps-e3486c526059">Chaos Engineering Traps</a>.’</p>
<div id="jp-post-flair" class="sharedaddy sd-like-enabled sd-sharing-enabled"><div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-twitter"><a rel="nofollow noopener noreferrer"
data-shared="sharing-twitter-8869"
class="share-twitter sd-button share-icon"
href="https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/?share=twitter"
target="_blank"
aria-labelledby="sharing-twitter-8869"
>
<span id="sharing-twitter-8869" hidden>Share on X (Opens in new window)</span>
<span>X</span>
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
data-shared="sharing-linkedin-8869"
class="share-linkedin sd-button share-icon"
href="https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/?share=linkedin"
target="_blank"
aria-labelledby="sharing-linkedin-8869"
>
<span id="sharing-linkedin-8869" hidden>Share on LinkedIn (Opens in new window)</span>
<span>LinkedIn</span>
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
data-shared="sharing-email-8869"
class="share-email sd-button share-icon"
href="mailto:?subject=%5BShared%20Post%5D%20Automating%20chaos%20experiments%20in%20production&#038;body=https%3A%2F%2Fblog.acolyer.org%2F2019%2F07%2F05%2Fautomating-chaos-experiments-in-production%2F&#038;share=email"
target="_blank"
aria-labelledby="sharing-email-8869"
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you&#039;re having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="8d14f05c7b" data-email-share-track-url="https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/?share=email">
<span id="sharing-email-8869" hidden>Email a link to a friend (Opens in new window)</span>
<span>Email</span>
</a></li><li class="share-print"><a rel="nofollow noopener noreferrer"
data-shared="sharing-print-8869"
class="share-print sd-button share-icon"
href="https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/#print?share=print"
target="_blank"
aria-labelledby="sharing-print-8869"
>
<span id="sharing-print-8869" hidden>Print (Opens in new window)</span>
<span>Print</span>
</a></li><li class="share-end"></li></ul></div></div></div><div class='sharedaddy sd-block sd-like jetpack-likes-widget-wrapper jetpack-likes-widget-unloaded' id='like-post-wrapper-23592848-8869-6aa49860992e2' data-src='//widgets.wp.com/likes/index.html?ver=20260912#blog_id=23592848&amp;post_id=8869&amp;origin=adriancolyer.wordpress.com&amp;obj_id=23592848-8869-6aa49860992e2&amp;domain=blog.acolyer.org' data-name='like-post-frame-23592848-8869-6aa49860992e2' data-title='Like or Reblog'><div class='likes-widget-placeholder post-likes-widget-placeholder' style='height: 55px;'><span class='button'><span>Like</span></span> <span class='loading'>Loading...</span></div><span class='sd-text-color'></span><a class='sd-link-color'></a></div>
<div id='jp-relatedposts' class='jp-relatedposts' >
<h3 class="jp-relatedposts-headline"><em>Related</em></h3>
</div></div> </div><!-- .entry-content -->
<footer class="entry-footer">
<span class="cat-links">
Posted in <a href="https://blog.acolyer.org/category/uncategorized/" rel="category tag">Uncategorized</a> </span>
<span class="tags-links">
<a href="https://blog.acolyer.org/tag/operations/" rel="tag">Operations</a><a href="https://blog.acolyer.org/tag/software-engineering/" rel="tag">Software Engineering</a> </span>
</footer><!-- .entry-footer -->
</article><!-- #post-## -->
<nav class="navigation post-navigation" aria-label="Posts">
<h2 class="screen-reader-text">Post navigation</h2>
<div class="nav-links"><div class="nav-previous"><a href="https://blog.acolyer.org/2019/07/03/one-sql-to-rule-them-all/" rel="prev"><span class="meta-nav">&lsaquo; Previous</span>One SQL to rule them all: an efficient and syntactically idiomatic approach to management of streams and&nbsp;tables</a></div><div class="nav-next"><a href="https://blog.acolyer.org/2019/07/08/software-engineering-for-machine-learning/" rel="next"><span class="meta-nav">Next &rsaquo;</span>Software engineering for machine learning: a case&nbsp;study</a></div></div>
</nav>
<div id="comments" class="comments-area">
<h2 class="comments-title">
2 thoughts on &ldquo;<span>Automating chaos experiments in&nbsp;production</span>&rdquo; </h2><!-- .comments-title -->
<ol class="comment-list">
<li id="comment-58944" class="pingback even thread-even depth-1">
<div class="comment-body">
Pingback: <a href="http://sreweekly.com/sre-weekly-issue-176/" class="url" rel="ugc external nofollow">SRE Weekly Issue #176 &#8211; SRE WEEKLY</a> </div>
</li><!-- #comment-## -->
<li id="comment-59239" class="pingback odd alt thread-odd thread-alt depth-1">
<div class="comment-body">
Pingback: <a href="https://blog.acolyer.org/2019/07/10/three-key-checklists-and-remedies-for-oces/" class="url" rel="ugc">Three key checklists and remedies for trustworthy analysis of online controlled experiments at scale &#8211; the morning paper</a> </div>
</li><!-- #comment-## -->
</ol><!-- .comment-list -->
<p class="no-comments">Comments are closed.</p>
</div><!-- #comments -->
</main><!-- #main -->
</div><!-- #primary -->
</div><!-- #content -->
<footer id="colophon" class="site-footer" role="contentinfo">
<div class="site-info-wrapper clear">
<div class="site-info">
<a href="https://wordpress.com/?ref=footer_blog" rel="nofollow">Blog at WordPress.com.</a>
</div><!-- .site-info -->
</div><!-- .site-info-wrapper -->
</footer><!-- #colophon -->
</div><!-- #page -->
<!-- -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/files/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/pub/penscratch-2/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script id="grofiles-cards-js" src="//0.gravatar.com/js/hovercards/hovercards.min.js?ver=202637d1133547e8a1f2801609f51cef12522abe7f6647d869e9fbc974fe356a37e613"></script>
<script id="wpgroho-js-extra">
var WPGroHo = {"my_hash":""};
//# sourceURL=wpgroho-js-extra
</script>
<script crossorigin='anonymous' type='text/javascript' src='/wp-content/mu-plugins/gravatar-hovercards/wpgroho.js?m=1610363240i'></script>
<script>
// Initialize and attach hovercards to all gravatars
( function() {
function init() {
if ( typeof Gravatar === 'undefined' ) {
return;
}
if ( typeof Gravatar.init !== 'function' ) {
return;
}
Gravatar.profile_cb = function ( hash, id ) {
WPGroHo.syncProfileData( hash, id );
};
Gravatar.my_hash = WPGroHo.my_hash;
Gravatar.init(
'body',
'#wp-admin-bar-my-account',
{
i18n: {
'Edit your profile →': 'Edit your profile →',
'View profile →': 'View profile →',
'Contact': 'Contact',
'Send money': 'Send money',
'Sorry, we are unable to load this Gravatar profile.': 'Sorry, we are unable to load this Gravatar profile.',
'Gravatar not found.': 'Gravatar not found.',
'Too Many Requests.': 'Too Many Requests.',
'Internal Server Error.': 'Internal Server Error.',
'Is this you?': 'Is this you?',
'Claim your free profile.': 'Claim your free profile.',
'Email': 'Email',
'Home Phone': 'Home Phone',
'Work Phone': 'Work Phone',
'Cell Phone': 'Cell Phone',
'Contact Form': 'Contact Form',
'Calendar': 'Calendar',
},
}
);
}
if ( document.readyState !== 'loading' ) {
init();
} else {
document.addEventListener( 'DOMContentLoaded', init );
}
} )();
</script>
<div style="display:none">
</div>
<div id="actionbar" dir="ltr" style="display: none;"
class="actnbr-pub-penscratch-2 actnbr-has-follow actnbr-has-actions">
<ul>
<li class="actnbr-btn actnbr-hidden">
<a class="actnbr-action actnbr-actn-reblog" href="">
<svg class="gridicon gridicons-reblog" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M22.086 9.914L20 7.828V18c0 1.105-.895 2-2 2h-7v-2h7V7.828l-2.086 2.086L14.5 8.5 19 4l4.5 4.5-1.414 1.414zM6 16.172V6h7V4H6c-1.105 0-2 .895-2 2v10.172l-2.086-2.086L.5 15.5 5 20l4.5-4.5-1.414-1.414L6 16.172z"/></g></svg><span>Reblog</span>
</a>
</li>
<li class="actnbr-btn actnbr-hidden">
<a class="actnbr-action actnbr-actn-follow " href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
<span>Subscribe</span>
</a>
<a class="actnbr-action actnbr-actn-following no-display" href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
<span>Subscribed</span>
</a>
<div class="actnbr-popover tip tip-top-left actnbr-notice" id="follow-bubble">
<div class="tip-arrow"></div>
<div class="tip-inner actnbr-follow-bubble">
<ul>
<li class="actnbr-sitename">
<a href="https://blog.acolyer.org">
<img loading='lazy' alt='' src='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png' srcset='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=75&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1.5x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=100&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 2x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=150&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 3x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=200&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 4x' class='avatar avatar-50' height='50' width='50' /> the morning paper </a>
</li>
<div class="actnbr-message no-display"></div>
<form method="post" action="https://subscribe.wordpress.com" accept-charset="utf-8" style="display: none;">
<div class="actnbr-follow-count">Join 3,125 other subscribers</div>
<div>
<input type="email" name="email" placeholder="Enter your email address" class="actnbr-email-field" aria-label="Enter your email address" />
</div>
<input type="hidden" name="action" value="subscribe" />
<input type="hidden" name="blog_id" value="23592848" />
<input type="hidden" name="source" value="https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/" />
<input type="hidden" name="sub-type" value="actionbar-follow" />
<input type="hidden" id="_wpnonce" name="_wpnonce" value="a0b9dafa84" /> <div class="actnbr-button-wrap">
<button type="submit" value="Sign me up">
Sign me up </button>
</div>
</form>
<li class="actnbr-login-nudge">
<div>
Already have a WordPress.com account? <a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fblog.acolyer.org%252F2019%252F07%252F05%252Fautomating-chaos-experiments-in-production%252F">Log in now.</a> </div>
</li>
</ul>
</div>
</div>
</li>
<li class="actnbr-ellipsis actnbr-hidden">
<svg class="gridicon gridicons-ellipsis" height="24" width="24" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M7 12c0 1.104-.896 2-2 2s-2-.896-2-2 .896-2 2-2 2 .896 2 2zm12-2c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2zm-7 0c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2z"/></g></svg> <div class="actnbr-popover tip tip-top-left actnbr-more">
<div class="tip-arrow"></div>
<div class="tip-inner">
<ul>
<li class="actnbr-sitename">
<a href="https://blog.acolyer.org">
<img loading='lazy' alt='' src='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png' srcset='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=75&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1.5x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=100&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 2x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=150&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 3x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=200&#038;d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 4x' class='avatar avatar-50' height='50' width='50' /> the morning paper </a>
</li>
<li class="actnbr-folded-follow">
<a class="actnbr-action actnbr-actn-follow " href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
<span>Subscribe</span>
</a>
<a class="actnbr-action actnbr-actn-following no-display" href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
<span>Subscribed</span>
</a>
</li>
<li class="actnbr-signup"><a href="https://wordpress.com/start/">Sign up</a></li>
<li class="actnbr-login"><a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fblog.acolyer.org%252F2019%252F07%252F05%252Fautomating-chaos-experiments-in-production%252F">Log in</a></li>
<li class="actnbr-shortlink">
<a href="https://wp.me/p1AZzO-2j3">
<span class="actnbr-shortlink__text">Copy shortlink</span>
<span class="actnbr-shortlink__icon"><svg class="gridicon gridicons-checkmark" height="16" width="16" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M9 19.414l-6.707-6.707 1.414-1.414L9 16.586 20.293 5.293l1.414 1.414"/></g></svg></span>
</a>
</li>
<li class="flb-report">
<a href="https://wordpress.com/abuse/?report_url=https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/" target="_blank" rel="noopener noreferrer">
Report this content </a>
</li>
<li class="actnbr-reader">
<a href="https://wordpress.com/reader/blogs/23592848/posts/8869">
View post in Reader </a>
</li>
<li class="actnbr-subs">
<a href="https://subscribe.wordpress.com/">Manage subscriptions</a>
</li>
<li class="actnbr-fold"><a href="">Collapse this bar</a></li>
</ul>
</div>
</div>
</li>
</ul>
</div>
<script>
window.addEventListener( "DOMContentLoaded", function( event ) {
var link = document.createElement( "link" );
link.href = "/wp-content/mu-plugins/actionbar/actionbar.css?v=20250116";
link.type = "text/css";
link.rel = "stylesheet";
document.head.appendChild( link );
var script = document.createElement( "script" );
script.src = "/wp-content/mu-plugins/actionbar/actionbar.js?v=20250204";
document.body.appendChild( script );
} );
</script>
<script type="text/javascript">
window.WPCOM_sharing_counts = {"https://blog.acolyer.org/2019/07/05/automating-chaos-experiments-in-production/":8869};
</script>
<script type="text/javascript">
(function () {
var wpcom_reblog = {
source: 'toolbar',
toggle_reblog_box_flair: function (obj_id, post_id) {
// Go to site selector. This will redirect to their blog if they only have one.
const postEndpoint = `https://wordpress.com/post`;
// Ideally we would use the permalink here, but fortunately this will be replaced with the
// post permalink in the editor.
const originalURL = `${ document.location.href }?page_id=${ post_id }`;
const url =
postEndpoint +
'?url=' +
encodeURIComponent( originalURL ) +
'&is_post_share=true' +
'&v=5';
const redirect = function () {
if (
! window.open( url, '_blank' )
) {
location.href = url;
}
};
if ( /Firefox/.test( navigator.userAgent ) ) {
setTimeout( redirect, 0 );
} else {
redirect();
}
},
};
window.wpcom_reblog = wpcom_reblog;
})();
</script>
<script id="coblocks-loader-js-extra">
var wpcom_coblocks_js = {"coblocks_masonry_js":"https://s2.wp.com/wp-content/plugins/coblocks/2.18.1-simple-rev.4/dist/js/coblocks-masonry.min.js?m=1681832297i","coblocks_lightbox_js":"https://s2.wp.com/wp-content/plugins/coblocks/2.18.1-simple-rev.4/dist/js/coblocks-lightbox.min.js?m=1681832297i","jquery_core_js":"/wp-includes/js/jquery/jquery.min.js","jquery_migrate_js":"/wp-includes/js/jquery/jquery-migrate.min.js","masonry_js":"/wp-includes/js/masonry.min.js","imagesloaded_js":"/wp-includes/js/imagesloaded.min.js"};
var coblocksLigthboxData = {"closeLabel":"Close Gallery","leftLabel":"Previous","rightLabel":"Next"};
//# sourceURL=coblocks-loader-js-extra
</script>
<script id="sharing-js-js-extra">
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":"1"};
//# sourceURL=sharing-js-js-extra
</script>
<script crossorigin='anonymous' type='text/javascript' src='/_static/??-eJyVkN1uwjAMRl8I1ysaUC4QjzKliWndJE6oE37efmxiWsUdd/6kcz7LxmsGm6SQFIwVcqgDi2JgT4rnSpVGIy7Q3Ey6wgX8R9rUh2S94rppu6YF5ZgDwUyX5hMda8HpH4KQjHt0RZbXvjJSfKzMtcdMonY2xY6w/rHFXHgwhdM70jK/oannDIHFwynZqnDi26u9eNNEJRvrnxljSoJfLBb7ysGhjmYmZ5y7/44sw/PyYzy0u26333x02+30Ddt6jhE='></script>
<script id="sharing-js-js-after">
window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
window.WPCOM_sharing_popups[ "twitter" ] = "menubar=1,resizable=1,width=600,height=350";
window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
window.WPCOM_sharing_popups[ "linkedin" ] = "menubar=1,resizable=1,width=580,height=450";
//# sourceURL=sharing-js-js-after
</script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/72x72/","ext":".png","svgUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/svg/","svgExt":".svg","source":{"concatemoji":"/wp-includes/js/wp-emoji-release.min.js?m=1783090810i&ver=7.1.1-alpha-63555"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=/wp-includes/js/wp-emoji-loader.min.js
</script>
<iframe src='https://widgets.wp.com/likes/master.html?ver=20260912#ver=20260912' scrolling='no' id='likes-master' name='likes-master' style='display:none;'></iframe>
<div id='likes-other-gravatars' role="dialog" aria-hidden="true" tabindex="-1"><div class="likes-text"><span>%d</span></div><ul class="wpl-avatars sd-like-gravatars"></ul></div>
<script src="//stats.wp.com/w.js?69" defer></script> <script type="text/javascript">
_tkq = window._tkq || [];
_stq = window._stq || [];
_tkq.push(['storeContext', {'blog_id':'23592848','blog_tz':'1','user_lang':'en','blog_lang':'en','user_id':'0'}]);
// Prevent sending pageview tracking from WP-Admin pages.
_stq.push(['view', {'blog':'23592848','v':'wpcom','tz':'1','user_id':'0','post':'8869','subd':'adriancolyer'}]);
_stq.push(['extra', {'crypt':'UE5tW3cvZGRDbzJxMWx6aVlydldVeXIvdUpWanJmOVg4Mys2XWxoN19+Lk5YJUp6QyVxQW02Q0lDM1lFMHBvZkEwd0d2K0ZtZnUxeixdRFUyQm84N1I4P2dYRVFiU2pmSFpJbDZBR0NxRU1DUDlhSkNfZ3Zafi4ramlUP0g5aXpYc1ZBUnxwV1kmNnFKU0tdZHJWXVY5fm1BVUh0b3Q1LDBBT3IlcCs/Lk0lZGhBTmUmNS9SQXY4TlQ3fFJKYU81RiZkNFE4VU0sLkxhTGdiWlY5PSVjUmRxbVNQMTEr'}]);
_stq.push([ 'clickTrackerInit', '23592848', '8869' ]);
</script>
<noscript><img src="https://pixel.wp.com/b.gif?v=noscript" style="height:1px;width:1px;overflow:hidden;position:absolute;bottom:1px;" alt="" /></noscript>
<meta id="bilmur" property="bilmur:data" content="" data-provider="wordpress.com" data-service="simple" data-site-tz="Europe/London" data-custom-props="{&quot;logged_in&quot;:&quot;0&quot;,&quot;wptheme&quot;:&quot;pub\/penscratch-2&quot;,&quot;wptheme_is_block&quot;:&quot;0&quot;}" >
<script defer src="/wp-content/js/bilmur.min.js?i=17&amp;m=202637"></script>
</body>
</html>