Files
nexus/sreweekly/articles/272/07-why-do-config-changes-keep-coming-up-in-major-incidents.html
2026-09-12 17:23:01 +08:00

1135 lines
107 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="profile" href="http://gmpg.org/xfn/11">
<title>Why do config changes keep coming up in major incidents? &#8211; Surfing Complexity</title>
<meta name='robots' content='max-image-preview:large' />
<!-- Async WordPress.com Remote Login -->
<script id="wpcom_remote_login_js">
var wpcom_remote_login_extra_auth = '';
function wpcom_remote_login_remove_dom_node_id( element_id ) {
var dom_node = document.getElementById( element_id );
if ( dom_node ) { dom_node.parentNode.removeChild( dom_node ); }
}
function wpcom_remote_login_remove_dom_node_classes( class_name ) {
var dom_nodes = document.querySelectorAll( '.' + class_name );
for ( var i = 0; i < dom_nodes.length; i++ ) {
dom_nodes[ i ].parentNode.removeChild( dom_nodes[ i ] );
}
}
function wpcom_remote_login_final_cleanup() {
wpcom_remote_login_remove_dom_node_classes( "wpcom_remote_login_msg" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_validate" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_js" );
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_iframe" );
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_styles" );
}
// Watch for messages back from the remote login
window.addEventListener( "message", function( e ) {
if ( e.origin === "https://r-login.wordpress.com" ) {
var data = {};
try {
data = JSON.parse( e.data );
} catch( e ) {
wpcom_remote_login_final_cleanup();
return;
}
if ( data.msg === 'LOGIN' ) {
// Clean up the login check iframe
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
var id_regex = new RegExp( /^[0-9]+$/ );
var token_regex = new RegExp( /^.*|.*|.*$/ );
if (
token_regex.test( data.token )
&& id_regex.test( data.wpcomid )
) {
// We have everything we need to ask for a login
var script = document.createElement( "script" );
script.setAttribute( "id", "wpcom_remote_login_validate" );
script.src = '/remote-login.php?wpcom_remote_login=validate'
+ '&wpcomid=' + data.wpcomid
+ '&token=' + encodeURIComponent( data.token )
+ '&host=' + window.location.protocol
+ '//' + window.location.hostname
+ '&postid=1894'
+ '&is_singular=1';
document.body.appendChild( script );
}
return;
}
// Safari ITP, not logged in, so redirect
if ( data.msg === 'LOGIN-REDIRECT' ) {
window.location = 'https://wordpress.com/log-in?redirect_to=' + window.location.href;
return;
}
// Safari ITP, storage access failed, remove the request
if ( data.msg === 'LOGIN-REMOVE' ) {
var css_zap = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } @media screen and ( max-width: 782px ) { html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } }';
var style_zap = document.createElement( 'style' );
style_zap.type = 'text/css';
style_zap.appendChild( document.createTextNode( css_zap ) );
document.body.appendChild( style_zap );
var e = document.getElementById( 'wpcom_request_access_iframe' );
e.parentNode.removeChild( e );
document.cookie = 'wordpress_com_login_access=denied; path=/; max-age=31536000';
return;
}
// Safari ITP
if ( data.msg === 'REQUEST_ACCESS' ) {
console.log( 'request access: safari' );
// Check ITP iframe enable/disable knob
if ( wpcom_remote_login_extra_auth !== 'safari_itp_iframe' ) {
return;
}
// If we are in a "private window" there is no ITP.
var private_window = false;
try {
var opendb = window.openDatabase( null, null, null, null );
} catch( e ) {
private_window = true;
}
if ( private_window ) {
console.log( 'private window' );
return;
}
var iframe = document.createElement( 'iframe' );
iframe.id = 'wpcom_request_access_iframe';
iframe.setAttribute( 'scrolling', 'no' );
iframe.setAttribute( 'sandbox', 'allow-storage-access-by-user-activation allow-scripts allow-same-origin allow-top-navigation-by-user-activation' );
iframe.src = 'https://r-login.wordpress.com/remote-login.php?wpcom_remote_login=request_access&origin=' + encodeURIComponent( data.origin ) + '&wpcomid=' + encodeURIComponent( data.wpcomid );
var css = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 46px !important; } * html body { margin-top: 46px !important; } @media screen and ( max-width: 660px ) { html { margin-top: 71px !important; } * html body { margin-top: 71px !important; } #wpcom_request_access_iframe { display: block; height: 71px !important; } } #wpcom_request_access_iframe { border: 0px; height: 46px; position: fixed; top: 0; left: 0; width: 100%; min-width: 100%; z-index: 99999; background: #23282d; } ';
var style = document.createElement( 'style' );
style.type = 'text/css';
style.id = 'wpcom_request_access_styles';
style.appendChild( document.createTextNode( css ) );
document.body.appendChild( style );
document.body.appendChild( iframe );
}
if ( data.msg === 'DONE' ) {
wpcom_remote_login_final_cleanup();
}
}
}, false );
// Inject the remote login iframe after the page has had a chance to load
// more critical resources
window.addEventListener( "DOMContentLoaded", function( e ) {
var iframe = document.createElement( "iframe" );
iframe.style.display = "none";
iframe.setAttribute( "scrolling", "no" );
iframe.setAttribute( "id", "wpcom_remote_login_key" );
iframe.src = "https://r-login.wordpress.com/remote-login.php"
+ "?wpcom_remote_login=key"
+ "&origin=aHR0cHM6Ly9zdXJmaW5nY29tcGxleGl0eS5ibG9n"
+ "&wpcomid=30291541"
+ "&time=" + Math.floor( Date.now() / 1000 );
document.body.appendChild( iframe );
}, false );
</script>
<link rel='dns-prefetch' href='//s2.wp.com' />
<link rel='dns-prefetch' href='//widgets.wp.com' />
<link rel='dns-prefetch' href='//s1.wp.com' />
<link rel='dns-prefetch' href='//s0.wp.com' />
<link rel="alternate" type="application/rss+xml" title="Surfing Complexity &raquo; Feed" href="https://surfingcomplexity.blog/feed/" />
<link rel="alternate" type="application/rss+xml" title="Surfing Complexity &raquo; Comments Feed" href="https://surfingcomplexity.blog/comments/feed/" />
<link rel="alternate" type="application/rss+xml" title="Surfing Complexity &raquo; Why do config changes keep coming up in major&nbsp;incidents? Comments Feed" href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/feed/" />
<script type="text/javascript">
/* <![CDATA[ */
function addLoadEvent(func) {
var oldonload = window.onload;
if (typeof window.onload != 'function') {
window.onload = func;
} else {
window.onload = function () {
oldonload();
func();
}
}
}
/* ]]> */
</script>
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-2-1' href='/_static/??-eJxtzN0KQEAQQOEXssaKcCHPsq1Jy+xPZjavL1JKLr86HTiSsjEIBgGfVaK8uMCwoiRjt8fAOYCPcyZk2JGM4KxSZPmotMwF/B/Jbfh+b1355EfdddWg26av1xMpeTR5&cssminify=yes' type='text/css' media='all' />
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/common.min.css */
</style>
<style id="wp-block-embed-inline-css">
.wp-block-embed.alignleft,.wp-block-embed.alignright,.wp-block[data-align=left]>[data-type="core/embed"],.wp-block[data-align=right]>[data-type="core/embed"]{max-width:360px;width:100%}.wp-block-embed.alignleft .wp-block-embed__wrapper,.wp-block-embed.alignright .wp-block-embed__wrapper,.wp-block[data-align=left]>[data-type="core/embed"] .wp-block-embed__wrapper,.wp-block[data-align=right]>[data-type="core/embed"] .wp-block-embed__wrapper{min-width:280px}.wp-block-cover .wp-block-embed{min-height:240px;min-width:320px}.wp-block-group.is-layout-flex .wp-block-embed{flex:1 1 0%;min-width:0}.wp-block-embed{overflow-wrap:break-word}.wp-block-embed :where(figcaption){margin-bottom:1em;margin-top:.5em}.wp-block-embed iframe{max-width:100%}.wp-block-embed__wrapper{position:relative}.wp-embed-responsive .wp-has-aspect-ratio .wp-block-embed__wrapper:before{content:"";display:block;padding-top:50%}.wp-embed-responsive .wp-has-aspect-ratio iframe{bottom:0;height:100%;left:0;position:absolute;right:0;top:0;width:100%}.wp-embed-responsive .wp-embed-aspect-21-9 .wp-block-embed__wrapper:before{padding-top:42.85%}.wp-embed-responsive .wp-embed-aspect-18-9 .wp-block-embed__wrapper:before{padding-top:50%}.wp-embed-responsive .wp-embed-aspect-16-9 .wp-block-embed__wrapper:before{padding-top:56.25%}.wp-embed-responsive .wp-embed-aspect-4-3 .wp-block-embed__wrapper:before{padding-top:75%}.wp-embed-responsive .wp-embed-aspect-1-1 .wp-block-embed__wrapper:before{padding-top:100%}.wp-embed-responsive .wp-embed-aspect-9-16 .wp-block-embed__wrapper:before{padding-top:177.77%}.wp-embed-responsive .wp-embed-aspect-1-2 .wp-block-embed__wrapper:before{padding-top:200%}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/embed/style.min.css */
</style>
<style id="wp-block-embed-theme-inline-css">
.wp-block-embed :where(figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-embed :where(figcaption){color:hsla(0,0%,100%,.65)}.wp-block-embed{margin:0 0 1em}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/embed/theme.min.css */
</style>
<style id="wp-block-quote-inline-css">
.wp-block-quote{box-sizing:border-box;overflow-wrap:break-word}.wp-block-quote.is-large:where(:not(.is-style-plain)),.wp-block-quote.is-style-large:where(:not(.is-style-plain)){margin-bottom:1em;padding:0 1em}.wp-block-quote.is-large:where(:not(.is-style-plain)) p,.wp-block-quote.is-style-large:where(:not(.is-style-plain)) p{font-size:1.5em;font-style:italic;line-height:1.6}.wp-block-quote.is-large:where(:not(.is-style-plain)) cite,.wp-block-quote.is-large:where(:not(.is-style-plain)) footer,.wp-block-quote.is-style-large:where(:not(.is-style-plain)) cite,.wp-block-quote.is-style-large:where(:not(.is-style-plain)) footer{font-size:1.125em;text-align:right}.wp-block-quote>cite{display:block}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/quote/style.min.css */
</style>
<style id="wp-block-quote-theme-inline-css">
.wp-block-quote{border-left:.25em solid;margin:0 0 1.75em;padding-left:1em}.wp-block-quote cite,.wp-block-quote footer{color:currentColor;font-size:.8125em;font-style:normal;position:relative}.wp-block-quote:where(.has-text-align-right){border-left:none;border-right:.25em solid;padding-left:0;padding-right:1em}.wp-block-quote:where(.has-text-align-center){border:none;padding-left:0}.wp-block-quote.is-large,.wp-block-quote.is-style-large,.wp-block-quote:where(.is-style-plain){border:none}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/quote/theme.min.css */
</style>
<style id="wp-block-heading-inline-css">
h1:where(.wp-block-heading).has-background,h2:where(.wp-block-heading).has-background,h3:where(.wp-block-heading).has-background,h4:where(.wp-block-heading).has-background,h5:where(.wp-block-heading).has-background,h6:where(.wp-block-heading).has-background{padding:1.25em 2.375em}h1.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h1.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h2.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h2.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h3.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h3.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h4.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h4.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h5.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h5.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h6.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h6.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]){rotate:180deg}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/heading/style.min.css */
</style>
<style id="wp-block-latest-posts-inline-css">
.wp-block-latest-posts{box-sizing:border-box}.wp-block-latest-posts.alignleft{margin-right:2em}.wp-block-latest-posts.alignright{margin-left:2em}.wp-block-latest-posts.wp-block-latest-posts__list{list-style:none}.wp-block-latest-posts.wp-block-latest-posts__list li{clear:both;overflow-wrap:break-word}.wp-block-latest-posts.is-grid:not(.is-layout-grid){display:flex;flex-wrap:wrap}.wp-block-latest-posts.is-grid:not(.is-layout-grid) li{margin:0 1.25em 1.25em 0;width:100%}@media (min-width:600px){.wp-block-latest-posts.columns-2:not(.is-layout-grid) li{width:calc(50% - .625em)}.wp-block-latest-posts.columns-2:not(.is-layout-grid) li:nth-child(2n){margin-right:0}.wp-block-latest-posts.columns-3:not(.is-layout-grid) li{width:calc(33.33333% - .83333em)}.wp-block-latest-posts.columns-3:not(.is-layout-grid) li:nth-child(3n){margin-right:0}.wp-block-latest-posts.columns-4:not(.is-layout-grid) li{width:calc(25% - .9375em)}.wp-block-latest-posts.columns-4:not(.is-layout-grid) li:nth-child(4n){margin-right:0}.wp-block-latest-posts.columns-5:not(.is-layout-grid) li{width:calc(20% - 1em)}.wp-block-latest-posts.columns-5:not(.is-layout-grid) li:nth-child(5n){margin-right:0}.wp-block-latest-posts.columns-6:not(.is-layout-grid) li{width:calc(16.66667% - 1.04167em)}.wp-block-latest-posts.columns-6:not(.is-layout-grid) li:nth-child(6n){margin-right:0}}@media (max-width:600px){.wp-block-latest-posts-is-layout-grid[class*=columns-]:not(.has-native-responsive-grid){grid-template-columns:1fr}}:root :where(.wp-block-latest-posts.is-grid){padding:0}:root :where(.wp-block-latest-posts.wp-block-latest-posts__list){padding-left:0}.wp-block-latest-posts__post-author,.wp-block-latest-posts__post-date{display:block;font-size:.8125em}.wp-block-latest-posts__post-excerpt,.wp-block-latest-posts__post-full-content{margin-bottom:1em;margin-top:.5em}.wp-block-latest-posts__featured-image a{display:inline-block}.wp-block-latest-posts__featured-image img{height:auto;max-width:100%;width:auto}.wp-block-latest-posts__featured-image.alignleft{float:left;margin-right:1em}.wp-block-latest-posts__featured-image.alignright{float:right;margin-left:1em}.wp-block-latest-posts__featured-image.aligncenter{margin-bottom:1em;text-align:center}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/latest-posts/style.min.css */
</style>
<style id="wp-block-list-inline-css">
ol,ul{box-sizing:border-box}:root :where(.wp-block-list.has-background){padding:1.25em 2.375em}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/list/style.min.css */
</style>
<style id="wp-block-paragraph-inline-css">
.is-small-text{font-size:.875em}.is-regular-text{font-size:1em}.is-large-text{font-size:2.25em}.is-larger-text{font-size:3em}.has-drop-cap:not(:focus):first-letter{float:left;font-size:8.4em;font-style:normal;font-weight:100;line-height:.68;margin:.05em .1em 0 0;text-transform:uppercase}body.rtl .has-drop-cap:not(:focus):first-letter{float:none;margin-left:.1em}p.has-drop-cap.has-background{overflow:hidden}:root :where(p.has-background){padding:1.25em 2.375em}:where(p.has-text-color:not(.has-link-color)) a{color:inherit}p.has-text-align-left[style*="writing-mode:vertical-lr"],p.has-text-align-right[style*="writing-mode:vertical-rl"]{rotate:180deg}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/paragraph/style.min.css */
</style>
<style id="wp-block-search-inline-css">
.wp-block-search__button{margin-left:10px;word-break:normal}.wp-block-search__button.has-icon{line-height:0}.wp-block-search__button svg{fill:currentColor;height:1.25em;min-height:24px;min-width:24px;vertical-align:text-bottom;width:1.25em}:where(.wp-block-search__button){border:1px solid #ccc;padding:6px 10px}.wp-block-search__inside-wrapper{display:flex;flex:auto;flex-wrap:nowrap;max-width:100%}.wp-block-search__label{width:100%}.wp-block-search.wp-block-search__button-only .wp-block-search__button{box-sizing:border-box;display:flex;flex-shrink:0;justify-content:center;margin-left:0;max-width:100%}.wp-block-search.wp-block-search__button-only .wp-block-search__inside-wrapper{min-width:0!important;transition-property:width}.wp-block-search.wp-block-search__button-only .wp-block-search__input{flex-basis:100%;transition-duration:.3s}.wp-block-search.wp-block-search__button-only.wp-block-search__searchfield-hidden,.wp-block-search.wp-block-search__button-only.wp-block-search__searchfield-hidden .wp-block-search__inside-wrapper{overflow:hidden}.wp-block-search.wp-block-search__button-only.wp-block-search__searchfield-hidden .wp-block-search__input{border-left-width:0!important;border-right-width:0!important;flex-basis:0;flex-grow:0;margin:0;min-width:0!important;padding-left:0!important;padding-right:0!important;width:0!important}:where(.wp-block-search__input){-webkit-appearance:initial;-moz-appearance:initial;appearance:none;border:1px solid #949494;flex-grow:1;font-family:inherit;font-size:inherit;font-style:inherit;font-weight:inherit;letter-spacing:inherit;line-height:inherit;margin-left:0;margin-right:0;min-width:3rem;padding:8px;text-decoration:unset!important;text-transform:inherit}:where(.wp-block-search__button-inside .wp-block-search__inside-wrapper){background-color:#fff;border:1px solid #949494;box-sizing:border-box;padding:4px}:where(.wp-block-search__button-inside .wp-block-search__inside-wrapper) .wp-block-search__input{border:none;border-radius:0;padding:0 4px}:where(.wp-block-search__button-inside .wp-block-search__inside-wrapper) .wp-block-search__input:focus{outline:none}:where(.wp-block-search__button-inside .wp-block-search__inside-wrapper) :where(.wp-block-search__button){padding:4px 8px}.wp-block-search.aligncenter .wp-block-search__inside-wrapper{margin:auto}.wp-block[data-align=right] .wp-block-search.wp-block-search__button-only .wp-block-search__inside-wrapper{float:right}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/search/style.min.css */
</style>
<style id="wp-block-search-theme-inline-css">
.wp-block-search .wp-block-search__label{font-weight:700}.wp-block-search__button{border:1px solid #ccc;padding:.375em .625em}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/search/theme.min.css */
</style>
<style id="wp-block-library-inline-css-extra">
.has-text-align-justify {
text-align:justify;
}
.has-text-align-justify{text-align:justify;}
/*# sourceURL=wp-block-library-inline-css */
</style>
<style id="classic-theme-styles-inline-css">
.wp-block-button__link{background-color:#32373c;border-radius:9999px;box-shadow:none;color:#fff;font-size:1.125em;padding:calc(.667em + 2px) calc(1.333em + 2px);text-decoration:none}.wp-block-file__button{background:#32373c;color:#fff}.wp-block-accordion-heading{margin:0}.wp-block-accordion-heading__toggle{background-color:inherit!important;color:inherit!important}.wp-block-accordion-heading__toggle:not(:focus-visible){outline:none}.wp-block-accordion-heading__toggle:focus,.wp-block-accordion-heading__toggle:hover{background-color:inherit!important;border:none;box-shadow:none;color:inherit;padding:var(--wp--preset--spacing--20,1em) 0;text-decoration:none}.wp-block-accordion-heading__toggle:focus-visible{outline:auto;outline-offset:0}.wp-block-tab:not(.has-text-color){color:inherit!important}.wp-block-tab:not(.has-background){background-color:inherit!important}.wp-block-tab:focus,.wp-block-tab:hover{text-decoration:none}.wp-block-tab:focus-visible{outline:auto;outline-offset:0}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.8.0/build/styles/block-library/classic.min.css */
</style>
<style id="coblocks-inline-format-inline-css">
.uppercase{text-transform:uppercase}
/*# sourceURL=coblocks-inline-format-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-12-1' href='/_static/??-eJx9jEEOAiEQBD/kQECjeDC+ZWEnijIwYWA3/l72op689KFTVXplCCU3zE1z6reYRYfiUwlP0VYZpwxIJE4IFRd10HOU9iFA2iuhCiI7/ROiDt9WxfETT20jCOc4YUIa2D9t5eGA91xRBMZS7ATtPkTZvCtdzNEZt7f2fHq8Afl7R7w=&cssminify=yes' type='text/css' media='all' />
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #fff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--color--medium-blue: #0087be;--wp--preset--color--bright-blue: #00aadc;--wp--preset--color--dark-gray: #4d4d4b;--wp--preset--color--light-gray: #b3b3b1;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--font-family--albert-sans: 'Albert Sans', sans-serif;--wp--preset--font-family--alegreya: Alegreya, serif;--wp--preset--font-family--arvo: Arvo, serif;--wp--preset--font-family--bodoni-moda: 'Bodoni Moda', serif;--wp--preset--font-family--bricolage-grotesque: 'Bricolage Grotesque', sans-serif;--wp--preset--font-family--cabin: Cabin, sans-serif;--wp--preset--font-family--chivo: Chivo, sans-serif;--wp--preset--font-family--commissioner: Commissioner, sans-serif;--wp--preset--font-family--cormorant: Cormorant, serif;--wp--preset--font-family--courier-prime: 'Courier Prime', monospace;--wp--preset--font-family--crimson-pro: 'Crimson Pro', serif;--wp--preset--font-family--dm-mono: 'DM Mono', monospace;--wp--preset--font-family--dm-sans: 'DM Sans', sans-serif;--wp--preset--font-family--dm-serif-display: 'DM Serif Display', serif;--wp--preset--font-family--domine: Domine, serif;--wp--preset--font-family--eb-garamond: 'EB Garamond', serif;--wp--preset--font-family--epilogue: Epilogue, sans-serif;--wp--preset--font-family--fahkwang: Fahkwang, sans-serif;--wp--preset--font-family--figtree: Figtree, sans-serif;--wp--preset--font-family--fira-sans: 'Fira Sans', sans-serif;--wp--preset--font-family--fjalla-one: 'Fjalla One', sans-serif;--wp--preset--font-family--fraunces: Fraunces, serif;--wp--preset--font-family--gabarito: Gabarito, system-ui;--wp--preset--font-family--ibm-plex-mono: 'IBM Plex Mono', monospace;--wp--preset--font-family--ibm-plex-sans: 'IBM Plex Sans', sans-serif;--wp--preset--font-family--ibarra-real-nova: 'Ibarra Real Nova', serif;--wp--preset--font-family--instrument-serif: 'Instrument Serif', serif;--wp--preset--font-family--inter: Inter, sans-serif;--wp--preset--font-family--josefin-sans: 'Josefin Sans', sans-serif;--wp--preset--font-family--jost: Jost, sans-serif;--wp--preset--font-family--libre-baskerville: 'Libre Baskerville', serif;--wp--preset--font-family--libre-franklin: 'Libre Franklin', sans-serif;--wp--preset--font-family--literata: Literata, serif;--wp--preset--font-family--lora: Lora, serif;--wp--preset--font-family--merriweather: Merriweather, serif;--wp--preset--font-family--montserrat: Montserrat, sans-serif;--wp--preset--font-family--newsreader: Newsreader, serif;--wp--preset--font-family--noto-sans-mono: 'Noto Sans Mono', sans-serif;--wp--preset--font-family--nunito: Nunito, sans-serif;--wp--preset--font-family--open-sans: 'Open Sans', sans-serif;--wp--preset--font-family--overpass: Overpass, sans-serif;--wp--preset--font-family--pt-serif: 'PT Serif', serif;--wp--preset--font-family--petrona: Petrona, serif;--wp--preset--font-family--piazzolla: Piazzolla, serif;--wp--preset--font-family--playfair-display: 'Playfair Display', serif;--wp--preset--font-family--plus-jakarta-sans: 'Plus Jakarta Sans', sans-serif;--wp--preset--font-family--poppins: Poppins, sans-serif;--wp--preset--font-family--raleway: Raleway, sans-serif;--wp--preset--font-family--roboto: Roboto, sans-serif;--wp--preset--font-family--roboto-slab: 'Roboto Slab', serif;--wp--preset--font-family--rubik: Rubik, sans-serif;--wp--preset--font-family--rufina: Rufina, serif;--wp--preset--font-family--sora: Sora, sans-serif;--wp--preset--font-family--source-sans-3: 'Source Sans 3', sans-serif;--wp--preset--font-family--source-serif-4: 'Source Serif 4', serif;--wp--preset--font-family--space-mono: 'Space Mono', monospace;--wp--preset--font-family--syne: Syne, sans-serif;--wp--preset--font-family--texturina: Texturina, serif;--wp--preset--font-family--urbanist: Urbanist, sans-serif;--wp--preset--font-family--work-sans: 'Work Sans', sans-serif;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-medium-blue-color{color: var(--wp--preset--color--medium-blue) !important;}.has-bright-blue-color{color: var(--wp--preset--color--bright-blue) !important;}.has-dark-gray-color{color: var(--wp--preset--color--dark-gray) !important;}.has-light-gray-color{color: var(--wp--preset--color--light-gray) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-medium-blue-background-color{background-color: var(--wp--preset--color--medium-blue) !important;}.has-bright-blue-background-color{background-color: var(--wp--preset--color--bright-blue) !important;}.has-dark-gray-background-color{background-color: var(--wp--preset--color--dark-gray) !important;}.has-light-gray-background-color{background-color: var(--wp--preset--color--light-gray) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-medium-blue-border-color{border-color: var(--wp--preset--color--medium-blue) !important;}.has-bright-blue-border-color{border-color: var(--wp--preset--color--bright-blue) !important;}.has-dark-gray-border-color{border-color: var(--wp--preset--color--dark-gray) !important;}.has-light-gray-border-color{border-color: var(--wp--preset--color--light-gray) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}.has-albert-sans-font-family{font-family: var(--wp--preset--font-family--albert-sans) !important;}.has-alegreya-font-family{font-family: var(--wp--preset--font-family--alegreya) !important;}.has-arvo-font-family{font-family: var(--wp--preset--font-family--arvo) !important;}.has-bodoni-moda-font-family{font-family: var(--wp--preset--font-family--bodoni-moda) !important;}.has-bricolage-grotesque-font-family{font-family: var(--wp--preset--font-family--bricolage-grotesque) !important;}.has-cabin-font-family{font-family: var(--wp--preset--font-family--cabin) !important;}.has-chivo-font-family{font-family: var(--wp--preset--font-family--chivo) !important;}.has-commissioner-font-family{font-family: var(--wp--preset--font-family--commissioner) !important;}.has-cormorant-font-family{font-family: var(--wp--preset--font-family--cormorant) !important;}.has-courier-prime-font-family{font-family: var(--wp--preset--font-family--courier-prime) !important;}.has-crimson-pro-font-family{font-family: var(--wp--preset--font-family--crimson-pro) !important;}.has-dm-mono-font-family{font-family: var(--wp--preset--font-family--dm-mono) !important;}.has-dm-sans-font-family{font-family: var(--wp--preset--font-family--dm-sans) !important;}.has-dm-serif-display-font-family{font-family: var(--wp--preset--font-family--dm-serif-display) !important;}.has-domine-font-family{font-family: var(--wp--preset--font-family--domine) !important;}.has-eb-garamond-font-family{font-family: var(--wp--preset--font-family--eb-garamond) !important;}.has-epilogue-font-family{font-family: var(--wp--preset--font-family--epilogue) !important;}.has-fahkwang-font-family{font-family: var(--wp--preset--font-family--fahkwang) !important;}.has-figtree-font-family{font-family: var(--wp--preset--font-family--figtree) !important;}.has-fira-sans-font-family{font-family: var(--wp--preset--font-family--fira-sans) !important;}.has-fjalla-one-font-family{font-family: var(--wp--preset--font-family--fjalla-one) !important;}.has-fraunces-font-family{font-family: var(--wp--preset--font-family--fraunces) !important;}.has-gabarito-font-family{font-family: var(--wp--preset--font-family--gabarito) !important;}.has-ibm-plex-mono-font-family{font-family: var(--wp--preset--font-family--ibm-plex-mono) !important;}.has-ibm-plex-sans-font-family{font-family: var(--wp--preset--font-family--ibm-plex-sans) !important;}.has-ibarra-real-nova-font-family{font-family: var(--wp--preset--font-family--ibarra-real-nova) !important;}.has-instrument-serif-font-family{font-family: var(--wp--preset--font-family--instrument-serif) !important;}.has-inter-font-family{font-family: var(--wp--preset--font-family--inter) !important;}.has-josefin-sans-font-family{font-family: var(--wp--preset--font-family--josefin-sans) !important;}.has-jost-font-family{font-family: var(--wp--preset--font-family--jost) !important;}.has-libre-baskerville-font-family{font-family: var(--wp--preset--font-family--libre-baskerville) !important;}.has-libre-franklin-font-family{font-family: var(--wp--preset--font-family--libre-franklin) !important;}.has-literata-font-family{font-family: var(--wp--preset--font-family--literata) !important;}.has-lora-font-family{font-family: var(--wp--preset--font-family--lora) !important;}.has-merriweather-font-family{font-family: var(--wp--preset--font-family--merriweather) !important;}.has-montserrat-font-family{font-family: var(--wp--preset--font-family--montserrat) !important;}.has-newsreader-font-family{font-family: var(--wp--preset--font-family--newsreader) !important;}.has-noto-sans-mono-font-family{font-family: var(--wp--preset--font-family--noto-sans-mono) !important;}.has-nunito-font-family{font-family: var(--wp--preset--font-family--nunito) !important;}.has-open-sans-font-family{font-family: var(--wp--preset--font-family--open-sans) !important;}.has-overpass-font-family{font-family: var(--wp--preset--font-family--overpass) !important;}.has-pt-serif-font-family{font-family: var(--wp--preset--font-family--pt-serif) !important;}.has-petrona-font-family{font-family: var(--wp--preset--font-family--petrona) !important;}.has-piazzolla-font-family{font-family: var(--wp--preset--font-family--piazzolla) !important;}.has-playfair-display-font-family{font-family: var(--wp--preset--font-family--playfair-display) !important;}.has-plus-jakarta-sans-font-family{font-family: var(--wp--preset--font-family--plus-jakarta-sans) !important;}.has-poppins-font-family{font-family: var(--wp--preset--font-family--poppins) !important;}.has-raleway-font-family{font-family: var(--wp--preset--font-family--raleway) !important;}.has-roboto-font-family{font-family: var(--wp--preset--font-family--roboto) !important;}.has-roboto-slab-font-family{font-family: var(--wp--preset--font-family--roboto-slab) !important;}.has-rubik-font-family{font-family: var(--wp--preset--font-family--rubik) !important;}.has-rufina-font-family{font-family: var(--wp--preset--font-family--rufina) !important;}.has-sora-font-family{font-family: var(--wp--preset--font-family--sora) !important;}.has-source-sans-3-font-family{font-family: var(--wp--preset--font-family--source-sans-3) !important;}.has-source-serif-4-font-family{font-family: var(--wp--preset--font-family--source-serif-4) !important;}.has-space-mono-font-family{font-family: var(--wp--preset--font-family--space-mono) !important;}.has-syne-font-family{font-family: var(--wp--preset--font-family--syne) !important;}.has-texturina-font-family{font-family: var(--wp--preset--font-family--texturina) !important;}.has-urbanist-font-family{font-family: var(--wp--preset--font-family--urbanist) !important;}.has-work-sans-font-family{font-family: var(--wp--preset--font-family--work-sans) !important;}
:where(.wp-block-latest-posts.is-layout-flex){gap: 1.25em;}:where(.wp-block-latest-posts.is-layout-grid){gap: 1.25em;}
/*# sourceURL=global-styles-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-14-1' href='/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/verbum-comments.css?m=1785880900i&cssminify=yes' type='text/css' media='all' />
<link rel='stylesheet' id='verbum-gutenberg-css-css' href='https://widgets.wp.com/verbum-block-editor/block-editor.css?ver=1771405207' media='all' />
<link crossorigin='anonymous' rel='stylesheet' id='all-css-16-1' href='/_static/??-eJydkMEKwjAQRH/IuFoEzUH8FEnjmqZNNiGbUPx70yJYUTz0sswc3szuwhiFDpSRMvgioivGEkOPOSo9vDxwIbha0mCQMNkK8G+51cwbWGTmDj0yxNKCpRtGrIOyqN5Z7jCJBjg/HK4BKwKtC3pYVTudM1eLMergvyIW30hYa0yVZu5823+QwSDqcirbQB9G3J2yaUIv/rw/nhopd/Ig+yckdJQO&cssminify=yes' type='text/css' media='all' />
<link crossorigin='anonymous' rel='stylesheet' id='print-css-17-1' href='/wp-content/mu-plugins/global-print/global-print.css?m=1465851035i&cssminify=yes' type='text/css' media='print' />
<style id="jetpack-global-styles-frontend-style-inline-css">
:root { --font-headings: unset; --font-base: unset; --font-headings-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif; --font-base-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;}
/*# sourceURL=jetpack-global-styles-frontend-style-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-20-1' href='/_static/??-eJyNjcsKAjEMRX/ImhmUURfip0inLW3GNCmmQfx7H7gRN+7ugcs5cGsuCPfEHaq5RpaRFZbUmw+XD4MaQ5VolBS0+GuKPsb7eyLndVBdwd+iM3KA2ZAiqAT05Eiy6Bf8KHtJ9RkvW8gks6fX4VSP426/mQ7TMA7LA38FSqA=&cssminify=yes' type='text/css' media='all' />
<script id="jetpack_related-posts-js-extra">
var related_posts_js_options = {"post_heading":"h4"};
//# sourceURL=jetpack_related-posts-js-extra
</script>
<script id="wpcom-actionbar-placeholder-js-extra">
var actionbardata = {"siteID":"30291541","postID":"1894","siteURL":"https://surfingcomplexity.blog","xhrURL":"https://surfingcomplexity.blog/wp-admin/admin-ajax.php","nonce":"7f5617a28c","isLoggedIn":"","statusMessage":"","subsEmailDefault":"instantly","proxyScriptUrl":"https://s0.wp.com/wp-content/js/wpcom-proxy-request.js?m=1513050504i&amp;ver=20211021","shortlink":"https://wp.me/p236df-uy","i18n":{"followedText":"New posts from this site will now appear in your \u003Ca href=\"https://wordpress.com/reader\"\u003EReader\u003C/a\u003E","foldBar":"Collapse this bar","unfoldBar":"Expand this bar","shortLinkCopied":"Shortlink copied to clipboard."}};
//# sourceURL=wpcom-actionbar-placeholder-js-extra
</script>
<script id="jetpack-mu-wpcom-settings-js-before">
var JETPACK_MU_WPCOM_SETTINGS = {"assetsUrl":"https://s1.wp.com/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/"};
//# sourceURL=jetpack-mu-wpcom-settings-js-before
</script>
<script crossorigin='anonymous' type='text/javascript' src='/_static/??/wp-content/mu-plugins/jetpack-plugin/sun/_inc/build/related-posts/related-posts.min.js,/wp-content/js/rlt-proxy.js?m=1755011788j'></script>
<script id="rlt-proxy-js-after">
rltInitialize( {"token":null,"iframeOrigins":["https:\/\/widgets.wp.com"]} );
//# sourceURL=rlt-proxy-js-after
</script>
<link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://lorinhochstein.wordpress.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress.com" />
<link rel="canonical" href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/" />
<link rel='shortlink' href='https://wp.me/p236df-uy' />
<link rel="alternate" type="application/json+oembed" href="https://public-api.wordpress.com/oembed/?format=json&amp;url=https%3A%2F%2Fsurfingcomplexity.blog%2F2021%2F05%2F29%2Fwhy-do-config-changes-keep-coming-up-in-major-incidents%2F&amp;for=wpcom-auto-discovery" /><link rel="alternate" type="application/xml+oembed" href="https://public-api.wordpress.com/oembed/?format=xml&amp;url=https%3A%2F%2Fsurfingcomplexity.blog%2F2021%2F05%2F29%2Fwhy-do-config-changes-keep-coming-up-in-major-incidents%2F&amp;for=wpcom-auto-discovery" />
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="Why do config changes keep coming up in major incidents?" />
<meta property="og:url" content="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/" />
<meta property="og:description" content="Recently, Vijay Chidambaram (a CS professor at UT Austin) asked me, &#8220;Why do so many outages involve configuration changes?&#8221; Me, a few years ago, making a similar observation I didn&amp;#821…" />
<meta property="article:published_time" content="2021-05-30T03:24:40+00:00" />
<meta property="article:modified_time" content="2021-05-30T03:24:40+00:00" />
<meta property="og:site_name" content="Surfing Complexity" />
<meta property="og:image" content="https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=120" />
<meta property="og:image:width" content="200" />
<meta property="og:image:height" content="200" />
<meta property="og:image:alt" content="" />
<meta property="og:locale" content="en_US" />
<meta property="article:publisher" content="https://www.facebook.com/WordPresscom" />
<meta name="twitter:text:title" content="Why do config changes keep coming up in major&nbsp;incidents?" />
<meta name="twitter:image" content="https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=120" />
<meta name="twitter:card" content="summary" />
<!-- End Jetpack Open Graph Tags -->
<link rel='openid.server' href='https://surfingcomplexity.blog/?openidserver=1' />
<link rel='openid.delegate' href='https://surfingcomplexity.blog/' />
<link rel="search" type="application/opensearchdescription+xml" href="https://surfingcomplexity.blog/osd.xml" title="Surfing Complexity" />
<link rel="search" type="application/opensearchdescription+xml" href="https://s1.wp.com/opensearch.xml" title="WordPress.com" />
<link rel="pingback" href="https://surfingcomplexity.blog/xmlrpc.php"> <style type="text/css">
.recentcomments a {
display: inline !important;
padding: 0 !important;
margin: 0 !important;
}
table.recentcommentsavatartop img.avatar, table.recentcommentsavatarend img.avatar {
border: 0px;
margin: 0;
}
table.recentcommentsavatartop a, table.recentcommentsavatarend a {
border: 0px !important;
background-color: transparent !important;
}
td.recentcommentsavatarend, td.recentcommentsavatartop {
padding: 0px 0px 1px 0px;
margin: 0px;
}
td.recentcommentstextend {
border: none !important;
padding: 0px 0px 2px 10px;
}
.rtl td.recentcommentstextend {
padding: 0px 10px 2px 0px;
}
td.recentcommentstexttop {
border: none;
padding: 0px 0px 0px 10px;
}
.rtl td.recentcommentstexttop {
padding: 0px 10px 0px 0px;
}
</style>
<meta name="description" content="Recently, Vijay Chidambaram (a CS professor at UT Austin) asked me, &quot;Why do so many outages involve configuration changes?&quot; https://twitter.com/norootcause/status/916383043933192192 Me, a few years ago, making a similar observation I didn&#039;t have a good explanation for him, and I still don&#039;t. I&#039;m using this post as an exercise of thinking out loud about possible explanations&hellip;" />
<link rel="icon" href="https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=32" sizes="32x32" />
<link rel="icon" href="https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=120" sizes="192x192" />
<link rel="apple-touch-icon" href="https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=120" />
<meta name="msapplication-TileImage" content="https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=120" />
</head>
<body class="wp-singular post-template-default single single-post postid-1894 single-format-standard wp-embed-responsive wp-theme-pubindependent-publisher-2 customizer-styles-applied has-sidebar jetpack-reblog-enabled">
<div id="page" class="hfeed site">
<a class="skip-link screen-reader-text" href="#content">Skip to content</a>
<div id="hero-header" class="site-hero-section">
<header id="masthead" class="site-header" role="banner">
<div class="inner">
<div class="site-branding">
<a class="site-logo-link" href="https://surfingcomplexity.blog/">
<img referrerpolicy="no-referrer" alt='Unknown&#039;s avatar' src='https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=80&#038;d=identicon&#038;r=G' srcset='https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=80&#038;d=identicon&#038;r=G 1x, https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=120&#038;d=identicon&#038;r=G 1.5x, https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=160&#038;d=identicon&#038;r=G 2x, https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=240&#038;d=identicon&#038;r=G 3x, https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=320&#038;d=identicon&#038;r=G 4x' class='avatar avatar-80 site-logo-image' height='80' width='80' loading='eager' decoding='async' /> </a><!-- .site-logo-link -->
<p class="site-title"><a href="https://surfingcomplexity.blog/" rel="home">Surfing Complexity</a></p>
<p class="site-description">Lorin Hochstein&#039;s ramblings about software, complex systems, and incidents.</p>
</div><!-- .site-branding -->
</div><!-- .inner -->
</header><!-- #masthead -->
</div>
<div id="content-wrapper" class="content-wrapper">
<div id="content" class="site-content">
<div id="primary" class="content-area">
<main id="main" class="site-main" role="main">
<article id="post-1894" class="post-1894 post type-post status-publish format-standard hentry category-software category-systems">
<header class="entry-header">
<h1 class="entry-title">Why do config changes keep coming up in major&nbsp;incidents?</h1> </header><!-- .entry-header --> <div class="entry-meta">
<span class="byline">
<a href="https://surfingcomplexity.blog/author/lorinh/" rel="author">Lorin Hochstein</a> </span>
<span class="cat-links">
<a href="https://surfingcomplexity.blog/category/software/" rel="category tag">software</a>, <a href="https://surfingcomplexity.blog/category/systems/" rel="category tag">systems</a> </span><!-- .cat-links -->
<span class="published-on">
<time class="entry-date published updated" datetime="2021-05-29T20:24:40-07:00">May 29, 2021</time> </span>
<span class="word-count">2 Minutes</span> </div><!-- .entry-meta -->
<div class="entry-content">
<p class="wp-block-paragraph">Recently, <a href="https://twitter.com/vj_chidambaram">Vijay Chidambaram</a> (a CS professor at UT Austin) asked me, &#8220;Why do so many outages involve configuration changes?&#8221;</p>
<figure class="wp-block-embed is-type-rich is-provider-twitter wp-block-embed-twitter"><div class="wp-block-embed__wrapper">
<div class="embed-twitter"><blockquote class="twitter-tweet" data-width="550" data-dnt="true"><p lang="fr" dir="ltr">Hypothesis: config changes are more dangerous than code changes.</p>&mdash; Lorin Hochstein (@norootcause) <a href="https://twitter.com/norootcause/status/916383043933192192?ref_src=twsrc%5Etfw">October 6, 2017</a></blockquote><script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script></div>
</div><figcaption>Me, a few years ago, making a similar observation</figcaption></figure>
<p class="wp-block-paragraph">I didn&#8217;t have a good explanation for him, and I still don&#8217;t. I&#8217;m using this post as an exercise of thinking out loud about possible explanations for this phenomenon.</p>
<h2 class="wp-block-heading">It&#8217;s an illusion</h2>
<p class="wp-block-paragraph">It might be that config changes are not somehow more dangerous, it just seems like they are. Perhaps we only notice the writeups where a config change is mentioned, but we don&#8217;t remember the writeups that don&#8217;t involve a config change. Or perhaps it&#8217;s a <em>base rate</em> illusion, where config changes tend to be involved in incidents more often than code changes simply because config changes are more common than code changes.</p>
<p class="wp-block-paragraph">I don&#8217;t believe this hypothesis: I think the config change effect is a real one.</p>
<h2 class="wp-block-heading">Config changes as second-class </h2>
<p class="wp-block-paragraph">In the <a href="https://surfingcomplexity.blog/2021/05/25/subverting-the-process/">recent Salesforce incident</a>, the <a href="https://help.salesforce.com/articleView?id=000358392&amp;mode=1&amp;type=1">writeup</a> noted that:</p>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p><em>For many of Salesforce’s systems, the deployment pipelines have built-in stagger and canary requirements that are automated. For Salesforce’s DNS systems, the automation and enforcement of staggering through technology is still being built. For this configuration change and script, the stagger process was still manual.  </em></p></blockquote>
<p class="wp-block-paragraph">If an organization has the ability to stage their changes across different domains, I&#8217;d wager heavily that they supported staged code deployments before they supported staged configuration change. That&#8217;s certainly true at Netflix, where Spinnaker had support for regional rollout of code changes well before it had support for regional rollout of config changes.</p>
<p class="wp-block-paragraph">This one feels like a real contributor to me. I&#8217;ve found that deployment tooling tends to support code changes better than config change: there&#8217;s just more engineering effort put into making code changes safer. </p>
<h2 class="wp-block-heading">Config changes are hard to stage</h2>
<p class="wp-block-paragraph">In the case of the Salesforce incident, the configuration change could theoretically have been staged. However, it may be that configuration changes by their nature are harder to roll out in a staged fashion. Configuration is more likely to be inherently global than code.</p>
<p class="wp-block-paragraph">I&#8217;m really not sure about this one. I have no sense as to how many config changes can be staged.</p>
<h2 class="wp-block-heading">Config changes are hard to test</h2>
<p class="wp-block-paragraph">Have you ever written a unit test for a configuration value? I haven&#8217;t. It might be that config-change related problems only manifest when deployed into a production environment, so you couldn&#8217;t catch them at a smaller scope like a unit test.</p>
<p class="wp-block-paragraph">I suspect this hypothesis plays a significant role as well. </p>
<h2 class="wp-block-heading">Mature systems are more config-driven</h2>
<p class="wp-block-paragraph">Perhaps the sort of systems that are involved in large-scale outages at big tech companies are the more mature, reliable systems. These are the types of software that have evolved over time to enable operators to control more of their behavior by specifying policy in configuration.</p>
<p class="wp-block-paragraph">This means that an operator is more likely to be able to achieve a desired behavior change via config versus code. And that sounds like a good thing. We all know that <em>hard-coding</em> things is bad, and changing code is dangerous. In the limit, we wouldn&#8217;t have to make any code changes at all to achieve the desired system behavior.</p>
<p class="wp-block-paragraph">So, perhaps the fact that config changes are more commonly implicated in large-scale outages is a sign of the maturity of the systems?</p>
<p class="wp-block-paragraph">I have no idea about this one. It seems like a clever hypothesis, but perhaps it&#8217;s too clever.</p>
<div id="jp-post-flair" class="sharedaddy sd-like-enabled sd-sharing-enabled"><div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-twitter"><a rel="nofollow noopener noreferrer"
data-shared="sharing-twitter-1894"
class="share-twitter sd-button share-icon"
href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/?share=twitter"
target="_blank"
aria-labelledby="sharing-twitter-1894"
>
<span id="sharing-twitter-1894" hidden>Share on X (Opens in new window)</span>
<span>X</span>
</a></li><li class="share-facebook"><a rel="nofollow noopener noreferrer"
data-shared="sharing-facebook-1894"
class="share-facebook sd-button share-icon"
href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/?share=facebook"
target="_blank"
aria-labelledby="sharing-facebook-1894"
>
<span id="sharing-facebook-1894" hidden>Share on Facebook (Opens in new window)</span>
<span>Facebook</span>
</a></li><li class="share-end"></li></ul></div></div></div><div class='sharedaddy sd-block sd-like jetpack-likes-widget-wrapper jetpack-likes-widget-unloaded' id='like-post-wrapper-30291541-1894-6a8777be190a0' data-src='//widgets.wp.com/likes/index.html?ver=20260820#blog_id=30291541&amp;post_id=1894&amp;origin=lorinhochstein.wordpress.com&amp;obj_id=30291541-1894-6a8777be190a0&amp;domain=surfingcomplexity.blog' data-name='like-post-frame-30291541-1894-6a8777be190a0' data-title='Like or Reblog'><div class='likes-widget-placeholder post-likes-widget-placeholder' style='height: 55px;'><span class='button'><span>Like</span></span> <span class='loading'>Loading...</span></div><span class='sd-text-color'></span><a class='sd-link-color'></a></div>
<div id='jp-relatedposts' class='jp-relatedposts' >
</div></div> </div><!-- .entry-content -->
<div class="entry-footer">
</div><!-- .entry-footer -->
<div class="entry-author-wrapper">
<div class="entry-author author-avatar-show">
<div class="author-avatar">
<img referrerpolicy="no-referrer" alt='Unknown&#039;s avatar' src='https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=80&#038;d=identicon&#038;r=G' srcset='https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=80&#038;d=identicon&#038;r=G 1x, https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=120&#038;d=identicon&#038;r=G 1.5x, https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=160&#038;d=identicon&#038;r=G 2x, https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=240&#038;d=identicon&#038;r=G 3x, https://0.gravatar.com/avatar/f2641f12e815a54896f8f2ac04660c52efb896e09131390ad2a6f2f5fca81432?s=320&#038;d=identicon&#038;r=G 4x' class='avatar avatar-80' height='80' width='80' decoding='async' /> </div><!-- .author-avatar -->
<div class="author-heading">
<h2 class="author-title">
Published by <span class="author-name">Lorin Hochstein</span> </h2>
</div><!-- .author-heading -->
<p class="author-bio">
<a class="author-link" href="https://surfingcomplexity.blog/author/lorinh/" rel="author">
View all posts by Lorin Hochstein </a>
</p><!-- .author-bio -->
</div><!-- .entry-auhtor -->
<div class="site-posted-on">
<strong>Published</strong>
<time class="entry-date published updated" datetime="2021-05-29T20:24:40-07:00">May 29, 2021</time> </div><!-- .site-posted-on -->
</div>
</article><!-- #post-## -->
<nav class="navigation post-navigation" aria-label="Posts">
<h2 class="screen-reader-text">Post navigation</h2>
<div class="nav-links"><div class="nav-previous"><a href="https://surfingcomplexity.blog/2021/05/25/subverting-the-process/" rel="prev"><span class="meta-nav screen-reader-text">Previous Post</span> Subverting the process</a></div><div class="nav-next"><a href="https://surfingcomplexity.blog/2021/05/29/objectives-and-constraints/" rel="next"><span class="meta-nav screen-reader-text">Next Post</span> Objectives and constraints</a></div></div>
</nav>
<div id="comments" class="comments-area">
<h2 class="comments-title">
5 thoughts on &ldquo;<span>Why do config changes keep coming up in major&nbsp;incidents?</span>&rdquo; </h2><!-- .comments-title -->
<ol class="comment-list">
<li id="comment-2949" class="pingback even thread-even depth-1">
<div class="comment-body">
Pingback: <a href="https://leskowsky.net/2021/05/31/links-27/" class="url" rel="ugc external nofollow">Links &#8211; Christian Leskowsky</a> </div>
</li><!-- #comment-## -->
<li id="comment-2957" class="comment odd alt thread-odd thread-alt depth-1">
<article id="div-comment-2957" class="comment-body">
<footer class="comment-meta">
<div class="comment-author vcard">
<img referrerpolicy="no-referrer" alt='Tom&#039;s avatar' src='https://1.gravatar.com/avatar/7fde0a4fe77c45177cde6b35025bc326e8c279a39fb7aa6d95e1cc30bd0093f0?s=48&#038;d=identicon&#038;r=G' srcset='https://1.gravatar.com/avatar/7fde0a4fe77c45177cde6b35025bc326e8c279a39fb7aa6d95e1cc30bd0093f0?s=48&#038;d=identicon&#038;r=G 1x, https://1.gravatar.com/avatar/7fde0a4fe77c45177cde6b35025bc326e8c279a39fb7aa6d95e1cc30bd0093f0?s=72&#038;d=identicon&#038;r=G 1.5x, https://1.gravatar.com/avatar/7fde0a4fe77c45177cde6b35025bc326e8c279a39fb7aa6d95e1cc30bd0093f0?s=96&#038;d=identicon&#038;r=G 2x, https://1.gravatar.com/avatar/7fde0a4fe77c45177cde6b35025bc326e8c279a39fb7aa6d95e1cc30bd0093f0?s=144&#038;d=identicon&#038;r=G 3x, https://1.gravatar.com/avatar/7fde0a4fe77c45177cde6b35025bc326e8c279a39fb7aa6d95e1cc30bd0093f0?s=192&#038;d=identicon&#038;r=G 4x' class='avatar avatar-48' height='48' width='48' decoding='async' /> <b class="fn">Tom</b> <span class="says">says:</span> </div><!-- .comment-author -->
<div class="comment-metadata">
<a href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/#comment-2957"><time datetime="2021-06-04T10:14:57-07:00">June 4, 2021 at 10:14 am</time></a> </div><!-- .comment-metadata -->
</footer><!-- .comment-meta -->
<div class="comment-content">
<p>Are feature flags changes considered config changes? If that is the case, any issue due to a code change behind a feature flag is likely to show up as confi change causing the issue. This is really a special case of your last reason.</p>
</div><!-- .comment-content -->
<div class="reply"><a rel="nofollow" class="comment-reply-link" href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/?replytocom=2957#respond" data-commentid="2957" data-postid="1894" data-belowelement="div-comment-2957" data-respondelement="respond" data-replyto="Reply to Tom" aria-label="Reply to Tom">Reply</a></div> </article><!-- .comment-body -->
</li><!-- #comment-## -->
<li id="comment-6158" class="comment even thread-even depth-1">
<article id="div-comment-6158" class="comment-body">
<footer class="comment-meta">
<div class="comment-author vcard">
<img referrerpolicy="no-referrer" alt='Sun Chasing&#039;s avatar' src='https://1.gravatar.com/avatar/4dd4ded1f37160221b6c94020c24641329504cb538a68bb409fef787e827bfea?s=48&#038;d=identicon&#038;r=G' srcset='https://1.gravatar.com/avatar/4dd4ded1f37160221b6c94020c24641329504cb538a68bb409fef787e827bfea?s=48&#038;d=identicon&#038;r=G 1x, https://1.gravatar.com/avatar/4dd4ded1f37160221b6c94020c24641329504cb538a68bb409fef787e827bfea?s=72&#038;d=identicon&#038;r=G 1.5x, https://1.gravatar.com/avatar/4dd4ded1f37160221b6c94020c24641329504cb538a68bb409fef787e827bfea?s=96&#038;d=identicon&#038;r=G 2x, https://1.gravatar.com/avatar/4dd4ded1f37160221b6c94020c24641329504cb538a68bb409fef787e827bfea?s=144&#038;d=identicon&#038;r=G 3x, https://1.gravatar.com/avatar/4dd4ded1f37160221b6c94020c24641329504cb538a68bb409fef787e827bfea?s=192&#038;d=identicon&#038;r=G 4x' class='avatar avatar-48' height='48' width='48' decoding='async' /> <b class="fn"><a href="https://www.chasingsuns.com/" class="url" rel="ugc external nofollow">Sun Chasing</a></b> <span class="says">says:</span> </div><!-- .comment-author -->
<div class="comment-metadata">
<a href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/#comment-6158"><time datetime="2022-09-30T17:01:25-07:00">September 30, 2022 at 5:01 pm</time></a> </div><!-- .comment-metadata -->
</footer><!-- .comment-meta -->
<div class="comment-content">
<p>Thanks for writingg</p>
</div><!-- .comment-content -->
<div class="reply"><a rel="nofollow" class="comment-reply-link" href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/?replytocom=6158#respond" data-commentid="6158" data-postid="1894" data-belowelement="div-comment-6158" data-respondelement="respond" data-replyto="Reply to Sun Chasing" aria-label="Reply to Sun Chasing">Reply</a></div> </article><!-- .comment-body -->
</li><!-- #comment-## -->
<li id="comment-9452" class="comment byuser comment-author-jeffgrigg odd alt thread-odd thread-alt depth-1">
<article id="div-comment-9452" class="comment-body">
<footer class="comment-meta">
<div class="comment-author vcard">
<img referrerpolicy="no-referrer" alt='Jeff Grigg&#039;s avatar' src='https://1.gravatar.com/avatar/d04bb3eb7ad33e927c2e4bd769d5002d1aa7777178b5932b411491fe03a88d57?s=48&#038;d=identicon&#038;r=G' srcset='https://1.gravatar.com/avatar/d04bb3eb7ad33e927c2e4bd769d5002d1aa7777178b5932b411491fe03a88d57?s=48&#038;d=identicon&#038;r=G 1x, https://1.gravatar.com/avatar/d04bb3eb7ad33e927c2e4bd769d5002d1aa7777178b5932b411491fe03a88d57?s=72&#038;d=identicon&#038;r=G 1.5x, https://1.gravatar.com/avatar/d04bb3eb7ad33e927c2e4bd769d5002d1aa7777178b5932b411491fe03a88d57?s=96&#038;d=identicon&#038;r=G 2x, https://1.gravatar.com/avatar/d04bb3eb7ad33e927c2e4bd769d5002d1aa7777178b5932b411491fe03a88d57?s=144&#038;d=identicon&#038;r=G 3x, https://1.gravatar.com/avatar/d04bb3eb7ad33e927c2e4bd769d5002d1aa7777178b5932b411491fe03a88d57?s=192&#038;d=identicon&#038;r=G 4x' class='avatar avatar-48' height='48' width='48' loading='lazy' decoding='async' /> <b class="fn"><a href="http://jeffgrigg.wordpress.com/" class="url" rel="ugc external nofollow">Jeff Grigg</a></b> <span class="says">says:</span> </div><!-- .comment-author -->
<div class="comment-metadata">
<a href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/#comment-9452"><time datetime="2024-07-21T17:29:57-07:00">July 21, 2024 at 5:29 pm</time></a> </div><!-- .comment-metadata -->
</footer><!-- .comment-meta -->
<div class="comment-content">
<p class="wp-block-paragraph">I have written xUnit tests against configuration files. So I&#8217;ve had to think about this and work out details.</p>
<p class="wp-block-paragraph">It&#8217;s essentially impossible to test production configurations completely without deploying them into production, because the difference between test and production environments is the differences in configuration (files).</p>
<p class="wp-block-paragraph">It helps to set up &#8220;near production&#8221; systems that test *most* of the production functionality.</p>
<p class="wp-block-paragraph">When you have multiple (or A Great Many!) production systems, it&#8217;s a good idea to roll out to some and see how well it works (or not) before rolling out to the vast majority.</p>
<p class="wp-block-paragraph">It really helps to have reliable low-level facilities to roll it back to the last known good state.</p>
<p>(Even that is sometimes not sufficient, as the some changes make the system *Vulnerable* to failure. So the failure may not occur or be visible until &#8220;N&#8221; changes later. As in the AT&amp;T long distance outage &#8212; due to an uninitialized variable in *extreme heavy load* conditions.)</p>
</div><!-- .comment-content -->
<div class="reply"><a rel="nofollow" class="comment-reply-link" href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/?replytocom=9452#respond" data-commentid="9452" data-postid="1894" data-belowelement="div-comment-9452" data-respondelement="respond" data-replyto="Reply to Jeff Grigg" aria-label="Reply to Jeff Grigg">Reply</a></div> </article><!-- .comment-body -->
</li><!-- #comment-## -->
<li id="comment-9454" class="comment even thread-even depth-1">
<article id="div-comment-9454" class="comment-body">
<footer class="comment-meta">
<div class="comment-author vcard">
<img referrerpolicy="no-referrer" alt='AJ&#039;s avatar' src='https://0.gravatar.com/avatar/6224838d10a1121a4ff7e8dd05b42a759c3fcfb5032744898c3e8665dd65a69d?s=48&#038;d=identicon&#038;r=G' srcset='https://0.gravatar.com/avatar/6224838d10a1121a4ff7e8dd05b42a759c3fcfb5032744898c3e8665dd65a69d?s=48&#038;d=identicon&#038;r=G 1x, https://0.gravatar.com/avatar/6224838d10a1121a4ff7e8dd05b42a759c3fcfb5032744898c3e8665dd65a69d?s=72&#038;d=identicon&#038;r=G 1.5x, https://0.gravatar.com/avatar/6224838d10a1121a4ff7e8dd05b42a759c3fcfb5032744898c3e8665dd65a69d?s=96&#038;d=identicon&#038;r=G 2x, https://0.gravatar.com/avatar/6224838d10a1121a4ff7e8dd05b42a759c3fcfb5032744898c3e8665dd65a69d?s=144&#038;d=identicon&#038;r=G 3x, https://0.gravatar.com/avatar/6224838d10a1121a4ff7e8dd05b42a759c3fcfb5032744898c3e8665dd65a69d?s=192&#038;d=identicon&#038;r=G 4x' class='avatar avatar-48' height='48' width='48' loading='lazy' decoding='async' /> <b class="fn">AJ</b> <span class="says">says:</span> </div><!-- .comment-author -->
<div class="comment-metadata">
<a href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/#comment-9454"><time datetime="2024-07-22T12:00:24-07:00">July 22, 2024 at 12:00 pm</time></a> </div><!-- .comment-metadata -->
</footer><!-- .comment-meta -->
<div class="comment-content">
<p class="wp-block-paragraph">I worked at a big, enterprise-y company where the higher-ups decided as much as possible needed to be moved into Vault to reduce time to deploy down to essentially `kill -HUP &#8230;`</p>
</div><!-- .comment-content -->
<div class="reply"><a rel="nofollow" class="comment-reply-link" href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/?replytocom=9454#respond" data-commentid="9454" data-postid="1894" data-belowelement="div-comment-9454" data-respondelement="respond" data-replyto="Reply to AJ" aria-label="Reply to AJ">Reply</a></div> </article><!-- .comment-body -->
</li><!-- #comment-## -->
</ol><!-- .comment-list -->
<div id="respond" class="comment-respond">
<h3 id="reply-title" class="comment-reply-title">Leave a comment <small><a rel="nofollow" id="cancel-comment-reply-link" href="/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/#respond" style="display:none;">Cancel reply</a></small></h3><form action="https://surfingcomplexity.blog/wp-comments-post.php" method="post" id="commentform" class="comment-form">
<div class="comment-form__verbum transparent"></div><div class="verbum-form-meta"><input type='hidden' name='comment_post_ID' value='1894' id='comment_post_ID' />
<input type='hidden' name='comment_parent' id='comment_parent' value='0' />
<input type="hidden" name="highlander_comment_nonce" id="highlander_comment_nonce" value="5684526f52" />
<input type="hidden" name="verbum_show_subscription_modal" value="" /></div><p style="display: none;"><input type="hidden" id="akismet_comment_nonce" name="akismet_comment_nonce" value="bac80327ff" /></p><p style="display: none !important;" class="akismet-fields-container" data-prefix="ak_"><label>&#916;<textarea name="ak_hp_textarea" cols="45" rows="8" maxlength="100"></textarea></label><input type="hidden" id="ak_js_1" name="ak_js" value="248"/><script>
document.getElementById( "ak_js_1" ).setAttribute( "value", ( new Date() ).getTime() );
</script>
</p></form> </div><!-- #respond -->
</div><!-- #comments -->
</main><!-- #main -->
</div><!-- #primary -->
<div id="secondary" class="widget-area" role="complementary">
<aside id="block-3" class="widget widget_block widget_search"><form role="search" method="get" action="https://surfingcomplexity.blog/" class="wp-block-search__button-outside wp-block-search__text-button wp-block-search" ><label class="wp-block-search__label" for="wp-block-search__input-1" >Search</label><div class="wp-block-search__inside-wrapper" ><input class="wp-block-search__input" id="wp-block-search__input-1" placeholder="" value="" type="search" name="s" required /><button aria-label="Search" class="wp-block-search__button wp-element-button" type="submit" >Search</button></div></form></aside><aside id="block-7" class="widget widget_block">
<ul class="wp-block-list">
<li><a href="https://lorinhochstein.org">lorinhochstein.org</a></li>
<li><a href="https://bsky.app/profile/norootcause.surfingcomplexity.com">Bluesky</a></li>
<li><a href="https://hachyderm.io/@norootcause">Mastodon</a></li>
</ul>
</aside><aside id="block-4" class="widget widget_block widget_recent_entries"><ul class="wp-block-latest-posts__list wp-block-latest-posts is-layout-flow wp-block-latest-posts-is-layout-flow"><li><a class="wp-block-latest-posts__post-title" href="https://surfingcomplexity.blog/2026/08/19/github-autoscaling-and-the-component-substitution-fallacy/">GitHub, autoscaling, and the component substitution&nbsp;fallacy</a></li>
<li><a class="wp-block-latest-posts__post-title" href="https://surfingcomplexity.blog/2026/08/18/tough-days-at-github-a-continuing-series/">Tough days at GitHub, a continuing&nbsp;series</a></li>
<li><a class="wp-block-latest-posts__post-title" href="https://surfingcomplexity.blog/2026/08/16/quick-thoughts-on-azure-regional-outage-from-july-23-26/">Quick thoughts on Azure Regional Outage from July 23,&nbsp;&#8217;26</a></li>
<li><a class="wp-block-latest-posts__post-title" href="https://surfingcomplexity.blog/2026/08/13/github-has-another-tough-day/">GitHub has another tough&nbsp;day</a></li>
<li><a class="wp-block-latest-posts__post-title" href="https://surfingcomplexity.blog/2026/08/02/traditional-versus-resilience-engineering-views/">Traditional versus resilience engineering&nbsp;views</a></li>
</ul></aside></div><!-- #secondary -->
</div><!-- #content -->
<footer id="colophon" class="site-footer" role="contentinfo">
<div class="site-info">
<a href="https://wordpress.com/?ref=footer_blog" rel="nofollow">Blog at WordPress.com.</a>
</div><!-- .site-info -->
</footer><!-- #colophon -->
</div><!-- #content-wrapper -->
</div><!-- #page -->
<!-- -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/files/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/pub/independent-publisher-2/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script id="grofiles-cards-js" src="//0.gravatar.com/js/hovercards/hovercards.min.js?ver=202634d1133547e8a1f2801609f51cef12522abe7f6647d869e9fbc974fe356a37e613"></script>
<script id="wpgroho-js-extra">
var WPGroHo = {"my_hash":""};
//# sourceURL=wpgroho-js-extra
</script>
<script crossorigin='anonymous' type='text/javascript' src='/wp-content/mu-plugins/gravatar-hovercards/wpgroho.js?m=1610363240i'></script>
<script>
// Initialize and attach hovercards to all gravatars
( function() {
function init() {
if ( typeof Gravatar === 'undefined' ) {
return;
}
if ( typeof Gravatar.init !== 'function' ) {
return;
}
Gravatar.profile_cb = function ( hash, id ) {
WPGroHo.syncProfileData( hash, id );
};
Gravatar.my_hash = WPGroHo.my_hash;
Gravatar.init(
'body',
'#wp-admin-bar-my-account',
{
i18n: {
'Edit your profile →': 'Edit your profile →',
'View profile →': 'View profile →',
'Contact': 'Contact',
'Send money': 'Send money',
'Sorry, we are unable to load this Gravatar profile.': 'Sorry, we are unable to load this Gravatar profile.',
'Gravatar not found.': 'Gravatar not found.',
'Too Many Requests.': 'Too Many Requests.',
'Internal Server Error.': 'Internal Server Error.',
'Is this you?': 'Is this you?',
'Claim your free profile.': 'Claim your free profile.',
'Email': 'Email',
'Home Phone': 'Home Phone',
'Work Phone': 'Work Phone',
'Cell Phone': 'Cell Phone',
'Contact Form': 'Contact Form',
'Calendar': 'Calendar',
},
}
);
}
if ( document.readyState !== 'loading' ) {
init();
} else {
document.addEventListener( 'DOMContentLoaded', init );
}
} )();
</script>
<div style="display:none">
<div class="grofile-hash-map-c690634d340d85fb40ac03668259ff1c">
</div>
<div class="grofile-hash-map-93a9d23750c5cae4bbb54a0714669cdd">
</div>
<div class="grofile-hash-map-c8e477b939eb85bf27bae0d2995a3e35">
</div>
<div class="grofile-hash-map-1d8556a4a66c932f05c672bca5adb414">
</div>
<div class="grofile-hash-map-4689526a4587be058d939f7a4c3f82cb">
</div>
</div>
<div id="actionbar" dir="ltr" style="display: none;"
class="actnbr-pub-independent-publisher-2 actnbr-has-follow actnbr-has-actions">
<ul>
<li class="actnbr-btn actnbr-hidden">
<a class="actnbr-action actnbr-actn-comment" href="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/#comments">
<svg class="gridicon gridicons-comment" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M12 16l-5 5v-5H5c-1.1 0-2-.9-2-2V5c0-1.1.9-2 2-2h14c1.1 0 2 .9 2 2v9c0 1.1-.9 2-2 2h-7z"/></g></svg> <span>Comment </span>
</a>
</li>
<li class="actnbr-btn actnbr-hidden">
<a class="actnbr-action actnbr-actn-reblog" href="">
<svg class="gridicon gridicons-reblog" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M22.086 9.914L20 7.828V18c0 1.105-.895 2-2 2h-7v-2h7V7.828l-2.086 2.086L14.5 8.5 19 4l4.5 4.5-1.414 1.414zM6 16.172V6h7V4H6c-1.105 0-2 .895-2 2v10.172l-2.086-2.086L.5 15.5 5 20l4.5-4.5-1.414-1.414L6 16.172z"/></g></svg><span>Reblog</span>
</a>
</li>
<li class="actnbr-btn actnbr-hidden">
<a class="actnbr-action actnbr-actn-follow " href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
<span>Subscribe</span>
</a>
<a class="actnbr-action actnbr-actn-following no-display" href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
<span>Subscribed</span>
</a>
<div class="actnbr-popover tip tip-top-left actnbr-notice" id="follow-bubble">
<div class="tip-arrow"></div>
<div class="tip-inner actnbr-follow-bubble">
<ul>
<li class="actnbr-sitename">
<a href="https://surfingcomplexity.blog">
<img loading='lazy' alt='' src='https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=50' srcset='https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=50 1x, https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=75 1.5x, https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=100 2x, https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=120 3x, https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=120 4x' class='avatar avatar-50' height='50' width='50' /> Surfing Complexity </a>
</li>
<div class="actnbr-message no-display"></div>
<form method="post" action="https://subscribe.wordpress.com" accept-charset="utf-8" style="display: none;">
<div class="actnbr-follow-count">Join 258 other subscribers</div>
<div>
<input type="email" name="email" placeholder="Enter your email address" class="actnbr-email-field" aria-label="Enter your email address" />
</div>
<input type="hidden" name="action" value="subscribe" />
<input type="hidden" name="blog_id" value="30291541" />
<input type="hidden" name="source" value="https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/" />
<input type="hidden" name="sub-type" value="actionbar-follow" />
<input type="hidden" id="_wpnonce" name="_wpnonce" value="1ee5613bc2" /> <div class="actnbr-button-wrap">
<button type="submit" value="Sign me up">
Sign me up </button>
</div>
</form>
<li class="actnbr-login-nudge">
<div>
Already have a WordPress.com account? <a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fsurfingcomplexity.blog%252F2021%252F05%252F29%252Fwhy-do-config-changes-keep-coming-up-in-major-incidents%252F">Log in now.</a> </div>
</li>
</ul>
</div>
</div>
</li>
<li class="actnbr-ellipsis actnbr-hidden">
<svg class="gridicon gridicons-ellipsis" height="24" width="24" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M7 12c0 1.104-.896 2-2 2s-2-.896-2-2 .896-2 2-2 2 .896 2 2zm12-2c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2zm-7 0c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2z"/></g></svg> <div class="actnbr-popover tip tip-top-left actnbr-more">
<div class="tip-arrow"></div>
<div class="tip-inner">
<ul>
<li class="actnbr-sitename">
<a href="https://surfingcomplexity.blog">
<img loading='lazy' alt='' src='https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=50' srcset='https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=50 1x, https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=75 1.5x, https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=100 2x, https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=120 3x, https://surfingcomplexity.blog/wp-content/uploads/2020/05/surfer_1f3c4.png?w=120 4x' class='avatar avatar-50' height='50' width='50' /> Surfing Complexity </a>
</li>
<li class="actnbr-folded-follow">
<a class="actnbr-action actnbr-actn-follow " href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
<span>Subscribe</span>
</a>
<a class="actnbr-action actnbr-actn-following no-display" href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
<span>Subscribed</span>
</a>
</li>
<li class="actnbr-signup"><a href="https://wordpress.com/start/">Sign up</a></li>
<li class="actnbr-login"><a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fsurfingcomplexity.blog%252F2021%252F05%252F29%252Fwhy-do-config-changes-keep-coming-up-in-major-incidents%252F">Log in</a></li>
<li class="actnbr-shortlink">
<a href="https://wp.me/p236df-uy">
<span class="actnbr-shortlink__text">Copy shortlink</span>
<span class="actnbr-shortlink__icon"><svg class="gridicon gridicons-checkmark" height="16" width="16" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M9 19.414l-6.707-6.707 1.414-1.414L9 16.586 20.293 5.293l1.414 1.414"/></g></svg></span>
</a>
</li>
<li class="flb-report">
<a href="https://wordpress.com/abuse/?report_url=https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/" target="_blank" rel="noopener noreferrer">
Report this content </a>
</li>
<li class="actnbr-reader">
<a href="https://wordpress.com/reader/blogs/30291541/posts/1894">
View post in Reader </a>
</li>
<li class="actnbr-subs">
<a href="https://subscribe.wordpress.com/">Manage subscriptions</a>
</li>
<li class="actnbr-fold"><a href="">Collapse this bar</a></li>
</ul>
</div>
</div>
</li>
</ul>
</div>
<script>
window.addEventListener( "DOMContentLoaded", function( event ) {
var link = document.createElement( "link" );
link.href = "/wp-content/mu-plugins/actionbar/actionbar.css?v=20250116";
link.type = "text/css";
link.rel = "stylesheet";
document.head.appendChild( link );
var script = document.createElement( "script" );
script.src = "/wp-content/mu-plugins/actionbar/actionbar.js?v=20250204";
document.body.appendChild( script );
} );
</script>
<script type="text/javascript">
window.WPCOM_sharing_counts = {"https://surfingcomplexity.blog/2021/05/29/why-do-config-changes-keep-coming-up-in-major-incidents/":1894};
</script>
<script type="text/javascript">
(function () {
var wpcom_reblog = {
source: 'toolbar',
toggle_reblog_box_flair: function (obj_id, post_id) {
// Go to site selector. This will redirect to their blog if they only have one.
const postEndpoint = `https://wordpress.com/post`;
// Ideally we would use the permalink here, but fortunately this will be replaced with the
// post permalink in the editor.
const originalURL = `${ document.location.href }?page_id=${ post_id }`;
const url =
postEndpoint +
'?url=' +
encodeURIComponent( originalURL ) +
'&is_post_share=true' +
'&v=5';
const redirect = function () {
if (
! window.open( url, '_blank' )
) {
location.href = url;
}
};
if ( /Firefox/.test( navigator.userAgent ) ) {
setTimeout( redirect, 0 );
} else {
redirect();
}
},
};
window.wpcom_reblog = wpcom_reblog;
})();
</script>
<script crossorigin='anonymous' type='text/javascript' src='/wp-content/mu-plugins/likes/queuehandler.js?m=1741961244i'></script>
<script id="wp-hooks-js" src="/wp-content/plugins/gutenberg-core/v23.8.0/build/scripts/hooks/index.min.js?m=1787137145i&#038;ver=f0f188028580e8dc1255"></script>
<script id="wp-i18n-js" src="/wp-content/plugins/gutenberg-core/v23.8.0/build/scripts/i18n/index.min.js?m=1787137145i&#038;ver=1dfe7db3940c23ea9216"></script>
<script id="wp-i18n-js-after">
wp.i18n.setLocaleData( { 'text direction\u0004ltr': [ 'ltr' ] } );
//# sourceURL=wp-i18n-js-after
</script>
<script id="verbum-settings-js-before">
window.VerbumComments = {"Log in or provide your name and email to leave a reply.":"Log in or provide your name and email to leave a reply.","Log in or provide your name and email to leave a comment.":"Log in or provide your name and email to leave a comment.","Receive web and mobile notifications for posts on this site.":"Receive web and mobile notifications for posts on this site.","Name":"Name","Email (address never made public)":"Email (address never made public)","Website (optional)":"Website (optional)","Leave a reply. (log in optional)":"Leave a reply. (log in optional)","Leave a comment. (log in optional)":"Leave a comment. (log in optional)","Log in to leave a reply.":"Log in to leave a reply.","Log in to leave a comment.":"Log in to leave a comment.","Logged in via %s":"Logged in via %s","Log out":"Log out","Your browser is blocking the cookies needed to log in and comment here. Allow cookies in your privacy settings, then reload the page.":"Your browser is blocking the cookies needed to log in and comment here. Allow cookies in your privacy settings, then reload the page.","Email":"Email","(Address never made public)":"(Address never made public)","Instantly":"Instantly","Daily":"Daily","Reply":"Reply","Comment":"Comment","WordPress":"WordPress","Weekly":"Weekly","Notify me of new posts":"Notify me of new posts","Email me new posts":"Email me new posts","Email me new comments":"Email me new comments","Cancel":"Cancel","Write a comment...":"Write a comment...","Write a reply...":"Write a reply...","Website":"Website","Optional":"Optional","We'll keep you in the loop!":"We'll keep you in the loop!","Loading your comment...":"Loading your comment...","Discover more from":"Discover more from Surfing Complexity","Subscribe now to keep reading and get access to the full archive.":"Subscribe now to keep reading and get access to the full archive.","Continue reading":"Continue reading","Never miss a beat!":"Never miss a beat!","Interested in getting blog post updates? Simply click the button below to stay in the loop!":"Interested in getting blog post updates? Simply click the button below to stay in the loop!","Enter your email address":"Enter your email address","Subscribe":"Subscribe","Comment sent successfully":"Comment sent successfully","Save my name, email, and website in this browser for the next time I comment.":"Save my name, email, and website in this browser for the next time I comment.","hovercardi18n":{"Edit your profile \u2192":"Edit your profile \u2192","View profile \u2192":"View profile \u2192","Contact":"Contact","Send money":"Send money","Gravatar not found.":"Gravatar not found.","This profile is private.":"This profile is private.","Too Many Requests.":"Too Many Requests.","Internal Server Error.":"Internal Server Error.","Sorry, we are unable to load this Gravatar profile.":"Sorry, we are unable to load this Gravatar profile."},"siteId":30291541,"postId":1894,"mustLogIn":false,"requireNameEmail":true,"commentRegistration":false,"connectURL":"https://lorinhochstein.wordpress.com/public.api/connect/?action=request\u0026domain=surfingcomplexity.blog","logoutURL":"https://lorinhochstein.wordpress.com/wp-login.php?action=logout\u0026_wpnonce=19fccdc82a","homeURL":"https://surfingcomplexity.blog/","subscribeToBlog":true,"subscribeToComment":true,"isJetpackCommentsLoggedIn":false,"jetpackUsername":"","jetpackUserId":0,"jetpackSignature":"","jetpackAvatar":"https://0.gravatar.com/avatar/?s=96\u0026amp;d=identicon\u0026amp;r=G","enableBlocks":true,"enableSubscriptionModal":true,"currentLocale":"en","isJetpackComments":false,"allowedBlocks":["core/paragraph","core/list","core/code","core/list-item","core/quote","core/image","core/embed","core/quote","core/code"],"embedNonce":"1bfff32791","verbumBundleUrl":"/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/features/verbum-comments/dist/index.js","isRTL":false,"vbeCacheBuster":1771405207,"iframeUniqueId":0,"colorScheme":false}
//# sourceURL=verbum-settings-js-before
</script>
<script data-wp-strategy="defer" defer id="verbum-dynamic-loader-js" src="/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/assets/dynamic-loader.js?m=1772522099i&#038;minify=false&#038;ver=8b5296f9195db92c42e4"></script>
<script id="sharing-js-js-extra">
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":"1"};
//# sourceURL=sharing-js-js-extra
</script>
<script id="jetpack-script-data-js-before">
window.JetpackScriptData = {"site":{"host":"wpcom","is_wpcom_platform":true}};
//# sourceURL=jetpack-script-data-js-before
</script>
<script crossorigin='anonymous' type='text/javascript' src='/_static/??-eJydkM1OxDAMhF+I1MAKuhfEo6y8idu6TdIodvbn7cmWdoU4IOASJePxN6PAORk7R6WooAMFEkjlCBwdJapHVFPfnmWgbJ5hFIh44h6V59iM8gB/3peJk/EcJ9PNtojp+LKBOFpfXEVUm51DuG1nSv7aBP5PWuWB2MxJzTlV4HdEKCb50nOsiaQJ7WSqtFjXAUiJ2+xwqhFzBiw6B1Rle99CEVKBY2Hv7uKa7FDxh2CcWAKp2TWPcLgV3oQuL2b3i9Jfui6Izx4yYCaHzl2XK8d+/cb38PbU7l/bXbt/accP4X3ACw=='></script>
<script id="sharing-js-js-after">
var windowOpen;
( function () {
var shareWindowName = 'wpcomtwitter-' + Math.random().toString( 36 ).slice( 2 );
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-twitter' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-twitter' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), shareWindowName, 'menubar=1,resizable=1,width=600,height=350' );
return false;
}
} );
} )();
var windowOpen;
( function () {
var shareWindowName = 'wpcomfacebook-' + Math.random().toString( 36 ).slice( 2 );
function matches( el, sel ) {
return !! (
el.matches && el.matches( sel ) ||
el.msMatchesSelector && el.msMatchesSelector( sel )
);
}
document.body.addEventListener( 'click', function ( event ) {
if ( ! event.target ) {
return;
}
var el;
if ( matches( event.target, 'a.share-facebook' ) ) {
el = event.target;
} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
el = event.target.parentNode;
}
if ( el ) {
event.preventDefault();
// If there's another sharing window open, close it.
if ( typeof windowOpen !== 'undefined' ) {
windowOpen.close();
}
windowOpen = window.open( el.getAttribute( 'href' ), shareWindowName, 'menubar=1,resizable=1,width=600,height=400' );
return false;
}
} );
} )();
//# sourceURL=sharing-js-js-after
</script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/72x72/","ext":".png","svgUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/svg/","svgExt":".svg","source":{"concatemoji":"/wp-includes/js/wp-emoji-release.min.js?m=1783090810i&ver=7.1-RC3-63319"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=/wp-includes/js/wp-emoji-loader.min.js
</script>
<iframe src='https://widgets.wp.com/likes/master.html?ver=20260820#ver=20260820' scrolling='no' id='likes-master' name='likes-master' style='display:none;'></iframe>
<div id='likes-other-gravatars' role="dialog" aria-hidden="true" tabindex="-1"><div class="likes-text"><span>%d</span></div><ul class="wpl-avatars sd-like-gravatars"></ul></div>
<script src="//stats.wp.com/w.js?69" defer></script> <script type="text/javascript">
_tkq = window._tkq || [];
_stq = window._stq || [];
_tkq.push(['storeContext', {'blog_id':'30291541','blog_tz':'-7','user_lang':'en','blog_lang':'en','user_id':'0'}]);
// Prevent sending pageview tracking from WP-Admin pages.
_stq.push(['view', {'blog':'30291541','v':'wpcom','tz':'-7','user_id':'0','post':'1894','subd':'lorinhochstein'}]);
_stq.push(['extra', {'crypt':'UE5tW3cvZGRTd1VXTHx0Q0pyYT89K3QrZXh+YlZCT3lNLEN2NzQ2Njh8QTJiL1BzZVh+MTlwZTc3TkdTOFRnU25kb3V1UXJpTUJadTA4dlFQcj9QP1g/emVIUXlmUXcrR2V3LkdaZH5TVWowZHctPVhHeiZsLnd6aUdrYjhkdl1hMWEvSjlCbDRrN2diajdBcXhrPUF5OHIwJVhMX01SU1hIanZxL1VEWkFNRlJUP35sMS8yRkdDa1c4ckpMaE0rK0NZJS5TcjBiYXRHaUYuQTMlJjlWZQ=='}]);
_stq.push([ 'clickTrackerInit', '30291541', '1894' ]);
</script>
<noscript><img src="https://pixel.wp.com/b.gif?v=noscript" style="height:1px;width:1px;overflow:hidden;position:absolute;bottom:1px;" alt="" /></noscript>
<meta id="bilmur" property="bilmur:data" content="" data-provider="wordpress.com" data-service="simple" data-site-tz="America/Los_Angeles" data-custom-props="{&quot;logged_in&quot;:&quot;0&quot;,&quot;wptheme&quot;:&quot;pub\/independent-publisher-2&quot;,&quot;wptheme_is_block&quot;:&quot;0&quot;}" >
<script defer src="/wp-content/js/bilmur.min.js?i=17&amp;m=202634"></script>
<script>
(function() {
'use strict';
const fetches = {};
const promises = {};
const urls = {
'wp-polyfill': '/wp-includes/js/dist/vendor/wp-polyfill.min.js?m=1786440387i&ver=3.15.0',
'verbum': '/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/verbum-comments.js?m=1785880900i&minify=false&ver=ecf5cfe8827907d3ea28'
};
const loaders = {
'verbum': () => {
fetchExternalScript('wp-polyfill');
fetchExternalScript('verbum');
promises['wp-polyfill'] = promises['wp-polyfill'] || loadWPScript('wp-polyfill');
promises['verbum'] = promises['verbum'] || promises['wp-polyfill'].then( () => loadWPScript('verbum') );
return promises['verbum'];
},
};
const scriptExtras = {
};
window.WP_Enqueue_Dynamic_Script = {
loadScript: (handle) => {
if (!loaders[handle]) {
console.error('WP_Enqueue_Dynamic_Script: unregistered script `' + handle + '`.');
}
return loaders[handle]();
}
};
function fetchExternalScript(handle) {
if (!urls[handle]) {
return Promise.resolve();
}
fetches[handle] = fetches[handle] || fetch(urls[handle], { mode: 'no-cors' });
return fetches[handle];
}
function runExtraScript(handle, type, index) {
const id = 'wp-enqueue-dynamic-script:' + handle + ':' + type + ':' + (index + 1);
const template = document.getElementById(id);
if (!template) {
return Promise.reject();
}
const script = document.createElement( 'script' );
script.innerHTML = template.innerHTML;
document.body.appendChild( script );
return Promise.resolve();
}
function loadExternalScript(handle) {
if (!urls[handle]) {
return Promise.resolve();
}
return fetches[handle].then(() => {
return new Promise((resolve, reject) => {
const script = document.createElement('script');
script.onload = () => resolve();
script.onerror = (e) => reject(e);
script.src = urls[handle];
document.body.appendChild(script);
});
});
}
function loadExtra(handle, pos) {
const count = (scriptExtras[handle] && scriptExtras[handle][pos]) || 0;
let promise = Promise.resolve();
for (let i = 0; i < count; i++) {
promise = promise.then(() => runExtraScript(handle, pos, i));
}
return promise;
}
function loadWPScript(handle) {
// Core loads scripts in this order. See: https://github.com/WordPress/WordPress/blob/a59eb9d39c4fcba834b70c9e8dfd64feeec10ba6/wp-includes/class-wp-scripts.php#L428.
return loadExtra(handle, 'translations')
.then(() => loadExtra(handle, 'before'))
.then(() => loadExternalScript(handle))
.then(() => loadExtra(handle, 'after'));
}
} )();
</script>
</body>
</html>