905 lines
96 KiB
HTML
905 lines
96 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<link rel="profile" href="http://gmpg.org/xfn/11">
|
||
|
||
<title>What bugs cause cloud production incidents? – the morning paper</title>
|
||
<meta name='robots' content='max-image-preview:large' />
|
||
|
||
<!-- Async WordPress.com Remote Login -->
|
||
<script id="wpcom_remote_login_js">
|
||
var wpcom_remote_login_extra_auth = '';
|
||
function wpcom_remote_login_remove_dom_node_id( element_id ) {
|
||
var dom_node = document.getElementById( element_id );
|
||
if ( dom_node ) { dom_node.parentNode.removeChild( dom_node ); }
|
||
}
|
||
function wpcom_remote_login_remove_dom_node_classes( class_name ) {
|
||
var dom_nodes = document.querySelectorAll( '.' + class_name );
|
||
for ( var i = 0; i < dom_nodes.length; i++ ) {
|
||
dom_nodes[ i ].parentNode.removeChild( dom_nodes[ i ] );
|
||
}
|
||
}
|
||
function wpcom_remote_login_final_cleanup() {
|
||
wpcom_remote_login_remove_dom_node_classes( "wpcom_remote_login_msg" );
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_validate" );
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_js" );
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_iframe" );
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_styles" );
|
||
}
|
||
|
||
// Watch for messages back from the remote login
|
||
window.addEventListener( "message", function( e ) {
|
||
if ( e.origin === "https://r-login.wordpress.com" ) {
|
||
var data = {};
|
||
try {
|
||
data = JSON.parse( e.data );
|
||
} catch( e ) {
|
||
wpcom_remote_login_final_cleanup();
|
||
return;
|
||
}
|
||
|
||
if ( data.msg === 'LOGIN' ) {
|
||
// Clean up the login check iframe
|
||
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
|
||
|
||
var id_regex = new RegExp( /^[0-9]+$/ );
|
||
var token_regex = new RegExp( /^.*|.*|.*$/ );
|
||
if (
|
||
token_regex.test( data.token )
|
||
&& id_regex.test( data.wpcomid )
|
||
) {
|
||
// We have everything we need to ask for a login
|
||
var script = document.createElement( "script" );
|
||
script.setAttribute( "id", "wpcom_remote_login_validate" );
|
||
script.src = '/remote-login.php?wpcom_remote_login=validate'
|
||
+ '&wpcomid=' + data.wpcomid
|
||
+ '&token=' + encodeURIComponent( data.token )
|
||
+ '&host=' + window.location.protocol
|
||
+ '//' + window.location.hostname
|
||
+ '&postid=8832'
|
||
+ '&is_singular=1';
|
||
document.body.appendChild( script );
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
// Safari ITP, not logged in, so redirect
|
||
if ( data.msg === 'LOGIN-REDIRECT' ) {
|
||
window.location = 'https://wordpress.com/log-in?redirect_to=' + window.location.href;
|
||
return;
|
||
}
|
||
|
||
// Safari ITP, storage access failed, remove the request
|
||
if ( data.msg === 'LOGIN-REMOVE' ) {
|
||
var css_zap = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } @media screen and ( max-width: 782px ) { html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } }';
|
||
var style_zap = document.createElement( 'style' );
|
||
style_zap.type = 'text/css';
|
||
style_zap.appendChild( document.createTextNode( css_zap ) );
|
||
document.body.appendChild( style_zap );
|
||
|
||
var e = document.getElementById( 'wpcom_request_access_iframe' );
|
||
e.parentNode.removeChild( e );
|
||
|
||
document.cookie = 'wordpress_com_login_access=denied; path=/; max-age=31536000';
|
||
|
||
return;
|
||
}
|
||
|
||
// Safari ITP
|
||
if ( data.msg === 'REQUEST_ACCESS' ) {
|
||
console.log( 'request access: safari' );
|
||
|
||
// Check ITP iframe enable/disable knob
|
||
if ( wpcom_remote_login_extra_auth !== 'safari_itp_iframe' ) {
|
||
return;
|
||
}
|
||
|
||
// If we are in a "private window" there is no ITP.
|
||
var private_window = false;
|
||
try {
|
||
var opendb = window.openDatabase( null, null, null, null );
|
||
} catch( e ) {
|
||
private_window = true;
|
||
}
|
||
|
||
if ( private_window ) {
|
||
console.log( 'private window' );
|
||
return;
|
||
}
|
||
|
||
var iframe = document.createElement( 'iframe' );
|
||
iframe.id = 'wpcom_request_access_iframe';
|
||
iframe.setAttribute( 'scrolling', 'no' );
|
||
iframe.setAttribute( 'sandbox', 'allow-storage-access-by-user-activation allow-scripts allow-same-origin allow-top-navigation-by-user-activation' );
|
||
iframe.src = 'https://r-login.wordpress.com/remote-login.php?wpcom_remote_login=request_access&origin=' + encodeURIComponent( data.origin ) + '&wpcomid=' + encodeURIComponent( data.wpcomid );
|
||
|
||
var css = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 46px !important; } * html body { margin-top: 46px !important; } @media screen and ( max-width: 660px ) { html { margin-top: 71px !important; } * html body { margin-top: 71px !important; } #wpcom_request_access_iframe { display: block; height: 71px !important; } } #wpcom_request_access_iframe { border: 0px; height: 46px; position: fixed; top: 0; left: 0; width: 100%; min-width: 100%; z-index: 99999; background: #23282d; } ';
|
||
|
||
var style = document.createElement( 'style' );
|
||
style.type = 'text/css';
|
||
style.id = 'wpcom_request_access_styles';
|
||
style.appendChild( document.createTextNode( css ) );
|
||
document.body.appendChild( style );
|
||
|
||
document.body.appendChild( iframe );
|
||
}
|
||
|
||
if ( data.msg === 'DONE' ) {
|
||
wpcom_remote_login_final_cleanup();
|
||
}
|
||
}
|
||
}, false );
|
||
|
||
// Inject the remote login iframe after the page has had a chance to load
|
||
// more critical resources
|
||
window.addEventListener( "DOMContentLoaded", function( e ) {
|
||
var iframe = document.createElement( "iframe" );
|
||
iframe.style.display = "none";
|
||
iframe.setAttribute( "scrolling", "no" );
|
||
iframe.setAttribute( "id", "wpcom_remote_login_key" );
|
||
iframe.src = "https://r-login.wordpress.com/remote-login.php"
|
||
+ "?wpcom_remote_login=key"
|
||
+ "&origin=aHR0cHM6Ly9ibG9nLmFjb2x5ZXIub3Jn"
|
||
+ "&wpcomid=23592848"
|
||
+ "&time=" + Math.floor( Date.now() / 1000 );
|
||
document.body.appendChild( iframe );
|
||
}, false );
|
||
</script>
|
||
<link rel='dns-prefetch' href='//s2.wp.com' />
|
||
<link rel='dns-prefetch' href='//fonts-api.wp.com' />
|
||
<link rel='dns-prefetch' href='//s1.wp.com' />
|
||
<link rel='dns-prefetch' href='//s0.wp.com' />
|
||
<link rel="alternate" type="application/rss+xml" title="the morning paper » Feed" href="https://blog.acolyer.org/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="the morning paper » Comments Feed" href="https://blog.acolyer.org/comments/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="the morning paper » What bugs cause cloud production incidents? Comments Feed" href="https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/feed/" />
|
||
<script type="text/javascript">
|
||
/* <![CDATA[ */
|
||
function addLoadEvent(func) {
|
||
var oldonload = window.onload;
|
||
if (typeof window.onload != 'function') {
|
||
window.onload = func;
|
||
} else {
|
||
window.onload = function () {
|
||
oldonload();
|
||
func();
|
||
}
|
||
}
|
||
}
|
||
/* ]]> */
|
||
</script>
|
||
<style id="wp-img-auto-sizes-contain-inline-css">
|
||
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
|
||
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
|
||
</style>
|
||
<link crossorigin='anonymous' rel='stylesheet' id='all-css-2-1' href='/_static/??-eJxtjEsKgDAMBS9kjbqwuBDPUtog9o9J8fpaEQSRBw8GhoEjC50iY2QIRWRf1i0SWOSstHsYQkr1TPFIsKNXjEbkRPyhVhM18J/0m8M3fFPVlzD3UnbTtWG0J2CTNNk=&cssminify=yes' type='text/css' media='all' />
|
||
<style id="wp-emoji-styles-inline-css">
|
||
|
||
img.wp-smiley, img.emoji {
|
||
display: inline !important;
|
||
border: none !important;
|
||
box-shadow: none !important;
|
||
height: 1em !important;
|
||
width: 1em !important;
|
||
margin: 0 0.07em !important;
|
||
vertical-align: -0.1em !important;
|
||
background: none !important;
|
||
padding: 0 !important;
|
||
}
|
||
/*# sourceURL=wp-emoji-styles-inline-css */
|
||
</style>
|
||
<style id="wp-block-library-inline-css">
|
||
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
|
||
|
||
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/common.min.css */
|
||
</style>
|
||
<style id="wp-block-quote-inline-css">
|
||
.wp-block-quote{box-sizing:border-box;overflow-wrap:break-word}.wp-block-quote.is-large:where(:not(.is-style-plain)),.wp-block-quote.is-style-large:where(:not(.is-style-plain)){margin-bottom:1em;padding:0 1em}.wp-block-quote.is-large:where(:not(.is-style-plain)) p,.wp-block-quote.is-style-large:where(:not(.is-style-plain)) p{font-size:1.5em;font-style:italic;line-height:1.6}.wp-block-quote.is-large:where(:not(.is-style-plain)) cite,.wp-block-quote.is-large:where(:not(.is-style-plain)) footer,.wp-block-quote.is-style-large:where(:not(.is-style-plain)) cite,.wp-block-quote.is-style-large:where(:not(.is-style-plain)) footer{font-size:1.125em;text-align:right}.wp-block-quote>cite{display:block}
|
||
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/quote/style.min.css */
|
||
</style>
|
||
<style id="wp-block-heading-inline-css">
|
||
h1:where(.wp-block-heading).has-background,h2:where(.wp-block-heading).has-background,h3:where(.wp-block-heading).has-background,h4:where(.wp-block-heading).has-background,h5:where(.wp-block-heading).has-background,h6:where(.wp-block-heading).has-background{padding:1.25em 2.375em}h1.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h1.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h2.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h2.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h3.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h3.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h4.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h4.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h5.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h5.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]),h6.has-text-align-left[style*=writing-mode]:where([style*=vertical-lr]),h6.has-text-align-right[style*=writing-mode]:where([style*=vertical-rl]){rotate:180deg}
|
||
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/heading/style.min.css */
|
||
</style>
|
||
<style id="wp-block-image-inline-css">
|
||
.wp-block-image>a,.wp-block-image>figure>a{display:inline-block}.wp-block-image img{box-sizing:border-box;height:auto;max-width:100%;vertical-align:bottom}@media not (prefers-reduced-motion){.wp-block-image img.hide{visibility:hidden}.wp-block-image img.show{animation:show-content-image .4s}}.wp-block-image[style*=border-radius] img,.wp-block-image[style*=border-radius]>a{border-radius:inherit}.wp-block-image.has-custom-border img{box-sizing:border-box}.wp-block-image.aligncenter{text-align:center}.wp-block-image.alignfull>a,.wp-block-image.alignwide>a{width:100%}.wp-block-image.alignfull img,.wp-block-image.alignwide img{height:auto;width:100%}.wp-block-image .aligncenter,.wp-block-image .alignleft,.wp-block-image .alignright,.wp-block-image.aligncenter,.wp-block-image.alignleft,.wp-block-image.alignright{display:table}.wp-block-image .aligncenter>figcaption,.wp-block-image .alignleft>figcaption,.wp-block-image .alignright>figcaption,.wp-block-image.aligncenter>figcaption,.wp-block-image.alignleft>figcaption,.wp-block-image.alignright>figcaption{caption-side:bottom;display:table-caption}.wp-block-image .alignleft{float:left;margin:.5em 1em .5em 0}.wp-block-image .alignright{float:right;margin:.5em 0 .5em 1em}.wp-block-image .aligncenter{margin-left:auto;margin-right:auto}.wp-block-image :where(figcaption){margin-bottom:1em;margin-top:.5em}.wp-block-image.is-style-circle-mask img{border-radius:9999px}@supports ((-webkit-mask-image:none) or (mask-image:none)) or (-webkit-mask-image:none){.wp-block-image.is-style-circle-mask img{border-radius:0;-webkit-mask-image:url('data:image/svg+xml;utf8,<svg viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg"><circle cx="50" cy="50" r="50"/></svg>');mask-image:url('data:image/svg+xml;utf8,<svg viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg"><circle cx="50" cy="50" r="50"/></svg>');mask-mode:alpha;-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain}}:root :where(.wp-block-image.is-style-rounded img,.wp-block-image .is-style-rounded img){border-radius:9999px}.wp-block-image figure{margin:0}.wp-lightbox-container{display:flex;flex-direction:column;position:relative}.wp-lightbox-container img{cursor:zoom-in}.wp-lightbox-container img:hover+button{opacity:1}.wp-lightbox-container button{align-items:center;backdrop-filter:blur(16px) saturate(180%);background-color:rgba(90,90,90,.25);border:none;border-radius:4px;cursor:zoom-in;display:flex;height:20px;justify-content:center;opacity:0;padding:0;position:absolute;right:16px;text-align:center;top:16px;width:20px;z-index:100}@media not (prefers-reduced-motion){.wp-lightbox-container button{transition:opacity .2s ease}}.wp-lightbox-container button:focus-visible{outline:3px auto rgba(90,90,90,.25);outline:3px auto -webkit-focus-ring-color;outline-offset:3px}.wp-lightbox-container button:hover{cursor:pointer;opacity:1}.wp-lightbox-container button:focus{opacity:1}.wp-lightbox-container button:focus,.wp-lightbox-container button:hover,.wp-lightbox-container button:not(:hover):not(:active):not(.has-background){background-color:rgba(90,90,90,.25);border:none}.wp-lightbox-overlay{box-sizing:border-box;cursor:zoom-out;height:100vh;left:0;overflow:hidden;position:fixed;top:0;visibility:hidden;width:100%;z-index:100000}.wp-lightbox-overlay .wp-lightbox-close-button{align-items:center;cursor:pointer;display:flex;font-family:inherit;gap:8px;justify-content:center;line-height:1;min-height:40px;min-width:40px;padding:0 4px;position:absolute;right:calc(env(safe-area-inset-right) + 16px);top:calc(env(safe-area-inset-top) + 16px);z-index:5000000}.wp-lightbox-overlay .wp-lightbox-close-button:focus,.wp-lightbox-overlay .wp-lightbox-close-button:hover,.wp-lightbox-overlay .wp-lightbox-close-button:not(:hover):not(:active):not(.has-background){background:none;border:none}.wp-lightbox-overlay .wp-lightbox-close-button:has(.wp-lightbox-close-text:not([hidden])) .wp-lightbox-close-icon svg{height:1em;width:1em}.wp-lightbox-overlay .wp-lightbox-close-icon svg{display:block}.wp-lightbox-overlay .wp-lightbox-navigation-button-next,.wp-lightbox-overlay .wp-lightbox-navigation-button-prev{align-items:center;bottom:16px;cursor:pointer;display:flex;font-family:inherit;gap:4px;justify-content:center;line-height:1;min-height:40px;min-width:40px;padding:0 8px;position:absolute;z-index:2000002}.wp-lightbox-overlay .wp-lightbox-navigation-button-next[hidden],.wp-lightbox-overlay .wp-lightbox-navigation-button-prev[hidden]{display:none}@media (min-width:960px){.wp-lightbox-overlay .wp-lightbox-navigation-button-next,.wp-lightbox-overlay .wp-lightbox-navigation-button-prev{bottom:50%;transform:translateY(-50%)}}.wp-lightbox-overlay .wp-lightbox-navigation-button-next:focus,.wp-lightbox-overlay .wp-lightbox-navigation-button-next:hover,.wp-lightbox-overlay .wp-lightbox-navigation-button-next:not(:hover):not(:active):not(.has-background),.wp-lightbox-overlay .wp-lightbox-navigation-button-prev:focus,.wp-lightbox-overlay .wp-lightbox-navigation-button-prev:hover,.wp-lightbox-overlay .wp-lightbox-navigation-button-prev:not(:hover):not(:active):not(.has-background){background:none;border:none;padding:0 8px}.wp-lightbox-overlay .wp-lightbox-navigation-button-next:has(.wp-lightbox-navigation-text:not([hidden])) .wp-lightbox-navigation-icon svg,.wp-lightbox-overlay .wp-lightbox-navigation-button-prev:has(.wp-lightbox-navigation-text:not([hidden])) .wp-lightbox-navigation-icon svg{display:block;height:1.5em;width:1.5em}.wp-lightbox-overlay .wp-lightbox-navigation-button-prev{left:calc(env(safe-area-inset-left) + 16px)}.wp-lightbox-overlay .wp-lightbox-navigation-button-next{right:calc(env(safe-area-inset-right) + 16px)}.wp-lightbox-overlay .wp-lightbox-navigation-icon svg{vertical-align:middle}.wp-lightbox-overlay .lightbox-image-container{height:var(--wp--lightbox-container-height);left:50%;overflow:hidden;position:absolute;top:50%;transform:translate(-50%,-50%);transform-origin:top left;width:var(--wp--lightbox-container-width);z-index:2000001}.wp-lightbox-overlay .wp-block-image{align-items:center;box-sizing:border-box;display:flex;height:100%;justify-content:center;margin:0;position:relative;transform-origin:0 0;width:100%;z-index:3000000}.wp-lightbox-overlay .wp-block-image img{height:var(--wp--lightbox-image-height);min-height:var(--wp--lightbox-image-height);min-width:var(--wp--lightbox-image-width);width:var(--wp--lightbox-image-width)}.wp-lightbox-overlay .wp-block-image figcaption{display:none}.wp-lightbox-overlay button{background:none;border:none}.wp-lightbox-overlay .scrim{background-color:#fff;height:100%;opacity:.9;position:absolute;width:100%;z-index:2000000}.wp-lightbox-overlay.active{visibility:visible}@media not (prefers-reduced-motion){.wp-lightbox-overlay.active{animation:turn-on-visibility .25s both}.wp-lightbox-overlay.active img{animation:turn-on-visibility .35s both}.wp-lightbox-overlay.show-closing-animation:not(.active){animation:turn-off-visibility .35s both}.wp-lightbox-overlay.show-closing-animation:not(.active) img{animation:turn-off-visibility .25s both}.wp-lightbox-overlay.zoom.active{animation:none;opacity:1;visibility:visible}.wp-lightbox-overlay.zoom.active .lightbox-image-container{animation:lightbox-zoom-in .4s}.wp-lightbox-overlay.zoom.active .lightbox-image-container img{animation:none}.wp-lightbox-overlay.zoom.active .scrim{animation:turn-on-visibility .4s forwards}.wp-lightbox-overlay.zoom.show-closing-animation:not(.active){animation:none}.wp-lightbox-overlay.zoom.show-closing-animation:not(.active) .lightbox-image-container{animation:lightbox-zoom-out .4s}.wp-lightbox-overlay.zoom.show-closing-animation:not(.active) .lightbox-image-container img{animation:none}.wp-lightbox-overlay.zoom.show-closing-animation:not(.active) .scrim{animation:turn-off-visibility .4s forwards}}@keyframes show-content-image{0%{visibility:hidden}99%{visibility:hidden}to{visibility:visible}}@keyframes turn-on-visibility{0%{opacity:0}to{opacity:1}}@keyframes turn-off-visibility{0%{opacity:1;visibility:visible}99%{opacity:0;visibility:visible}to{opacity:0;visibility:hidden}}@keyframes lightbox-zoom-in{0%{transform:translate(calc((-100vw + var(--wp--lightbox-scrollbar-width))/2 + var(--wp--lightbox-initial-left-position)),calc(-50vh + var(--wp--lightbox-initial-top-position))) scale(var(--wp--lightbox-scale))}to{transform:translate(-50%,-50%) scale(1)}}@keyframes lightbox-zoom-out{0%{transform:translate(-50%,-50%) scale(1);visibility:visible}99%{visibility:visible}to{transform:translate(calc((-100vw + var(--wp--lightbox-scrollbar-width))/2 + var(--wp--lightbox-initial-left-position)),calc(-50vh + var(--wp--lightbox-initial-top-position))) scale(var(--wp--lightbox-scale));visibility:hidden}}
|
||
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/image/style.min.css */
|
||
</style>
|
||
<style id="wp-block-list-inline-css">
|
||
ol,ul{box-sizing:border-box}:root :where(.wp-block-list.has-background){padding:1.25em 2.375em}
|
||
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/list/style.min.css */
|
||
</style>
|
||
<style id="wp-block-paragraph-inline-css">
|
||
.is-small-text{font-size:.875em}.is-regular-text{font-size:1em}.is-large-text{font-size:2.25em}.is-larger-text{font-size:3em}.has-drop-cap:not(:focus):first-letter{float:left;font-size:8.4em;font-style:normal;font-weight:100;line-height:.68;margin:.05em .1em 0 0;text-transform:uppercase}body.rtl .has-drop-cap:not(:focus):first-letter{float:none;margin-left:.1em}p.has-drop-cap.has-background{overflow:hidden}:root :where(p.has-background){padding:1.25em 2.375em}:where(p.has-text-color:not(.has-link-color)) a{color:inherit}p.has-text-align-left[style*="writing-mode:vertical-lr"],p.has-text-align-right[style*="writing-mode:vertical-rl"]{rotate:180deg}
|
||
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/paragraph/style.min.css */
|
||
</style>
|
||
<style id="wp-block-library-inline-css-extra">
|
||
.has-text-align-justify {
|
||
text-align:justify;
|
||
}
|
||
.has-text-align-justify{text-align:justify;}
|
||
/*# sourceURL=wp-block-library-inline-css */
|
||
</style>
|
||
|
||
<style id="classic-theme-styles-inline-css">
|
||
.wp-block-button__link{background-color:#32373c;border-radius:9999px;box-shadow:none;color:#fff;font-size:1.125em;padding:calc(.667em + 2px) calc(1.333em + 2px);text-decoration:none}.wp-block-file__button{background:#32373c;color:#fff}.wp-block-accordion-heading{margin:0}.wp-block-accordion-heading__toggle{background-color:inherit!important;color:inherit!important}.wp-block-accordion-heading__toggle:not(:focus-visible){outline:none}.wp-block-accordion-heading__toggle:focus,.wp-block-accordion-heading__toggle:hover{background-color:inherit!important;border:none;box-shadow:none;color:inherit;padding:var(--wp--preset--spacing--20,1em) 0;text-decoration:none}.wp-block-accordion-heading__toggle:focus-visible{outline:auto;outline-offset:0}.wp-block-tab:not(.has-text-color){color:inherit!important}.wp-block-tab:not(.has-background){background-color:inherit!important}.wp-block-tab:focus,.wp-block-tab:hover{text-decoration:none}.wp-block-tab:focus-visible{outline:auto;outline-offset:0}
|
||
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/classic.min.css */
|
||
</style>
|
||
|
||
<style id="coblocks-inline-format-inline-css">
|
||
.uppercase{text-transform:uppercase}
|
||
/*# sourceURL=coblocks-inline-format-inline-css */
|
||
</style>
|
||
<link crossorigin='anonymous' rel='stylesheet' id='all-css-12-1' href='/_static/??-eJx9jEEOAiEQBD/kQECjeDC+ZWEnijIwYWA3/l72op689KFTVXplCCU3zE1z6reYRYfiUwlP0VYZpwxIJE4IFRd10HOU9iFA2iuhCiI7/ROiDt9WxfETT20jCOc4YUIa2D9t5eGA91xRBMZS7ATtPkTZvCtdzNEZt7f2fHq8Afl7R7w=&cssminify=yes' type='text/css' media='all' />
|
||
<style id="global-styles-inline-css">
|
||
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #fff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--color--dark-green: #1c7c7c;--wp--preset--color--dark-gray: #666;--wp--preset--color--medium-gray: #999;--wp--preset--color--light-gray: #eee;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--font-family--albert-sans: 'Albert Sans', sans-serif;--wp--preset--font-family--alegreya: Alegreya, serif;--wp--preset--font-family--arvo: Arvo, serif;--wp--preset--font-family--bodoni-moda: 'Bodoni Moda', serif;--wp--preset--font-family--bricolage-grotesque: 'Bricolage Grotesque', sans-serif;--wp--preset--font-family--cabin: Cabin, sans-serif;--wp--preset--font-family--chivo: Chivo, sans-serif;--wp--preset--font-family--commissioner: Commissioner, sans-serif;--wp--preset--font-family--cormorant: Cormorant, serif;--wp--preset--font-family--courier-prime: 'Courier Prime', monospace;--wp--preset--font-family--crimson-pro: 'Crimson Pro', serif;--wp--preset--font-family--dm-mono: 'DM Mono', monospace;--wp--preset--font-family--dm-sans: 'DM Sans', sans-serif;--wp--preset--font-family--dm-serif-display: 'DM Serif Display', serif;--wp--preset--font-family--domine: Domine, serif;--wp--preset--font-family--eb-garamond: 'EB Garamond', serif;--wp--preset--font-family--epilogue: Epilogue, sans-serif;--wp--preset--font-family--fahkwang: Fahkwang, sans-serif;--wp--preset--font-family--figtree: Figtree, sans-serif;--wp--preset--font-family--fira-sans: 'Fira Sans', sans-serif;--wp--preset--font-family--fjalla-one: 'Fjalla One', sans-serif;--wp--preset--font-family--fraunces: Fraunces, serif;--wp--preset--font-family--gabarito: Gabarito, system-ui;--wp--preset--font-family--ibm-plex-mono: 'IBM Plex Mono', monospace;--wp--preset--font-family--ibm-plex-sans: 'IBM Plex Sans', sans-serif;--wp--preset--font-family--ibarra-real-nova: 'Ibarra Real Nova', serif;--wp--preset--font-family--instrument-serif: 'Instrument Serif', serif;--wp--preset--font-family--inter: Inter, sans-serif;--wp--preset--font-family--josefin-sans: 'Josefin Sans', sans-serif;--wp--preset--font-family--jost: Jost, sans-serif;--wp--preset--font-family--libre-baskerville: 'Libre Baskerville', serif;--wp--preset--font-family--libre-franklin: 'Libre Franklin', sans-serif;--wp--preset--font-family--literata: Literata, serif;--wp--preset--font-family--lora: Lora, serif;--wp--preset--font-family--merriweather: Merriweather, serif;--wp--preset--font-family--montserrat: Montserrat, sans-serif;--wp--preset--font-family--newsreader: Newsreader, serif;--wp--preset--font-family--noto-sans-mono: 'Noto Sans Mono', sans-serif;--wp--preset--font-family--nunito: Nunito, sans-serif;--wp--preset--font-family--open-sans: 'Open Sans', sans-serif;--wp--preset--font-family--overpass: Overpass, sans-serif;--wp--preset--font-family--pt-serif: 'PT Serif', serif;--wp--preset--font-family--petrona: Petrona, serif;--wp--preset--font-family--piazzolla: Piazzolla, serif;--wp--preset--font-family--playfair-display: 'Playfair Display', serif;--wp--preset--font-family--plus-jakarta-sans: 'Plus Jakarta Sans', sans-serif;--wp--preset--font-family--poppins: Poppins, sans-serif;--wp--preset--font-family--raleway: Raleway, sans-serif;--wp--preset--font-family--roboto: Roboto, sans-serif;--wp--preset--font-family--roboto-slab: 'Roboto Slab', serif;--wp--preset--font-family--rubik: Rubik, sans-serif;--wp--preset--font-family--rufina: Rufina, serif;--wp--preset--font-family--sora: Sora, sans-serif;--wp--preset--font-family--source-sans-3: 'Source Sans 3', sans-serif;--wp--preset--font-family--source-serif-4: 'Source Serif 4', serif;--wp--preset--font-family--space-mono: 'Space Mono', monospace;--wp--preset--font-family--syne: Syne, sans-serif;--wp--preset--font-family--texturina: Texturina, serif;--wp--preset--font-family--urbanist: Urbanist, sans-serif;--wp--preset--font-family--work-sans: 'Work Sans', sans-serif;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-color{color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-color{color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-color{color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-color{color: var(--wp--preset--color--light-gray) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-background-color{background-color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-background-color{background-color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-background-color{background-color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-background-color{background-color: var(--wp--preset--color--light-gray) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-green-border-color{border-color: var(--wp--preset--color--dark-green) !important;}.has-dark-gray-border-color{border-color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-border-color{border-color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-border-color{border-color: var(--wp--preset--color--light-gray) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}.has-albert-sans-font-family{font-family: var(--wp--preset--font-family--albert-sans) !important;}.has-alegreya-font-family{font-family: var(--wp--preset--font-family--alegreya) !important;}.has-arvo-font-family{font-family: var(--wp--preset--font-family--arvo) !important;}.has-bodoni-moda-font-family{font-family: var(--wp--preset--font-family--bodoni-moda) !important;}.has-bricolage-grotesque-font-family{font-family: var(--wp--preset--font-family--bricolage-grotesque) !important;}.has-cabin-font-family{font-family: var(--wp--preset--font-family--cabin) !important;}.has-chivo-font-family{font-family: var(--wp--preset--font-family--chivo) !important;}.has-commissioner-font-family{font-family: var(--wp--preset--font-family--commissioner) !important;}.has-cormorant-font-family{font-family: var(--wp--preset--font-family--cormorant) !important;}.has-courier-prime-font-family{font-family: var(--wp--preset--font-family--courier-prime) !important;}.has-crimson-pro-font-family{font-family: var(--wp--preset--font-family--crimson-pro) !important;}.has-dm-mono-font-family{font-family: var(--wp--preset--font-family--dm-mono) !important;}.has-dm-sans-font-family{font-family: var(--wp--preset--font-family--dm-sans) !important;}.has-dm-serif-display-font-family{font-family: var(--wp--preset--font-family--dm-serif-display) !important;}.has-domine-font-family{font-family: var(--wp--preset--font-family--domine) !important;}.has-eb-garamond-font-family{font-family: var(--wp--preset--font-family--eb-garamond) !important;}.has-epilogue-font-family{font-family: var(--wp--preset--font-family--epilogue) !important;}.has-fahkwang-font-family{font-family: var(--wp--preset--font-family--fahkwang) !important;}.has-figtree-font-family{font-family: var(--wp--preset--font-family--figtree) !important;}.has-fira-sans-font-family{font-family: var(--wp--preset--font-family--fira-sans) !important;}.has-fjalla-one-font-family{font-family: var(--wp--preset--font-family--fjalla-one) !important;}.has-fraunces-font-family{font-family: var(--wp--preset--font-family--fraunces) !important;}.has-gabarito-font-family{font-family: var(--wp--preset--font-family--gabarito) !important;}.has-ibm-plex-mono-font-family{font-family: var(--wp--preset--font-family--ibm-plex-mono) !important;}.has-ibm-plex-sans-font-family{font-family: var(--wp--preset--font-family--ibm-plex-sans) !important;}.has-ibarra-real-nova-font-family{font-family: var(--wp--preset--font-family--ibarra-real-nova) !important;}.has-instrument-serif-font-family{font-family: var(--wp--preset--font-family--instrument-serif) !important;}.has-inter-font-family{font-family: var(--wp--preset--font-family--inter) !important;}.has-josefin-sans-font-family{font-family: var(--wp--preset--font-family--josefin-sans) !important;}.has-jost-font-family{font-family: var(--wp--preset--font-family--jost) !important;}.has-libre-baskerville-font-family{font-family: var(--wp--preset--font-family--libre-baskerville) !important;}.has-libre-franklin-font-family{font-family: var(--wp--preset--font-family--libre-franklin) !important;}.has-literata-font-family{font-family: var(--wp--preset--font-family--literata) !important;}.has-lora-font-family{font-family: var(--wp--preset--font-family--lora) !important;}.has-merriweather-font-family{font-family: var(--wp--preset--font-family--merriweather) !important;}.has-montserrat-font-family{font-family: var(--wp--preset--font-family--montserrat) !important;}.has-newsreader-font-family{font-family: var(--wp--preset--font-family--newsreader) !important;}.has-noto-sans-mono-font-family{font-family: var(--wp--preset--font-family--noto-sans-mono) !important;}.has-nunito-font-family{font-family: var(--wp--preset--font-family--nunito) !important;}.has-open-sans-font-family{font-family: var(--wp--preset--font-family--open-sans) !important;}.has-overpass-font-family{font-family: var(--wp--preset--font-family--overpass) !important;}.has-pt-serif-font-family{font-family: var(--wp--preset--font-family--pt-serif) !important;}.has-petrona-font-family{font-family: var(--wp--preset--font-family--petrona) !important;}.has-piazzolla-font-family{font-family: var(--wp--preset--font-family--piazzolla) !important;}.has-playfair-display-font-family{font-family: var(--wp--preset--font-family--playfair-display) !important;}.has-plus-jakarta-sans-font-family{font-family: var(--wp--preset--font-family--plus-jakarta-sans) !important;}.has-poppins-font-family{font-family: var(--wp--preset--font-family--poppins) !important;}.has-raleway-font-family{font-family: var(--wp--preset--font-family--raleway) !important;}.has-roboto-font-family{font-family: var(--wp--preset--font-family--roboto) !important;}.has-roboto-slab-font-family{font-family: var(--wp--preset--font-family--roboto-slab) !important;}.has-rubik-font-family{font-family: var(--wp--preset--font-family--rubik) !important;}.has-rufina-font-family{font-family: var(--wp--preset--font-family--rufina) !important;}.has-sora-font-family{font-family: var(--wp--preset--font-family--sora) !important;}.has-source-sans-3-font-family{font-family: var(--wp--preset--font-family--source-sans-3) !important;}.has-source-serif-4-font-family{font-family: var(--wp--preset--font-family--source-serif-4) !important;}.has-space-mono-font-family{font-family: var(--wp--preset--font-family--space-mono) !important;}.has-syne-font-family{font-family: var(--wp--preset--font-family--syne) !important;}.has-texturina-font-family{font-family: var(--wp--preset--font-family--texturina) !important;}.has-urbanist-font-family{font-family: var(--wp--preset--font-family--urbanist) !important;}.has-work-sans-font-family{font-family: var(--wp--preset--font-family--work-sans) !important;}
|
||
/*# sourceURL=global-styles-inline-css */
|
||
</style>
|
||
|
||
<link crossorigin='anonymous' rel='stylesheet' id='all-css-14-1' href='/_static/??/wp-content/themes/pub/penscratch-2/css/reset.css,/wp-content/themes/pub/penscratch-2/style.css?m=1779214100j&cssminify=yes' type='text/css' media='all' />
|
||
<style id="penscratch-2-style-inline-css">
|
||
.cat-links, .byline .author, .group-blog .byline + .sep, .single .byline + .sep { clip: rect(1px, 1px, 1px, 1px); height: 1px; position: absolute; overflow: hidden; width: 1px; }
|
||
/*# sourceURL=penscratch-2-style-inline-css */
|
||
</style>
|
||
<link rel='stylesheet' id='penscratch-2-fonts-css' href='https://fonts-api.wp.com/css?family=Roboto+Slab%3A300%2C400%2C700&subset=latin%2Clatin-ext' media='all' />
|
||
<link crossorigin='anonymous' rel='stylesheet' id='all-css-16-1' href='/_static/??-eJyNjsEKAjEMRH/ImO0iLB7Eb+mG2K22aWmyLP691YuIIN7mHd7M4FaBihiLoS2cWbGuM1YWpeaNFhiRVHFOhW6673GHfyhRCNXuiWGrVPKXl1eoaQ1RFBv37tBjeA298ZcUuEB/5C0W+QC4JB/bUz3nk5um4+gObhiuD4S6VWY=&cssminify=yes' type='text/css' media='all' />
|
||
<link crossorigin='anonymous' rel='stylesheet' id='print-css-17-1' href='/wp-content/mu-plugins/global-print/global-print.css?m=1465851035i&cssminify=yes' type='text/css' media='print' />
|
||
<style id="jetpack-global-styles-frontend-style-inline-css">
|
||
:root { --font-headings: unset; --font-base: unset; --font-headings-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif; --font-base-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;}
|
||
/*# sourceURL=jetpack-global-styles-frontend-style-inline-css */
|
||
</style>
|
||
<link crossorigin='anonymous' rel='stylesheet' id='all-css-20-1' href='/_static/??-eJyNjcEKAjEMRH/ImhVlqwfxU6TbhrZrmhTTIv69u+JFvHgZ5sHwBh7VeOGG3KB0U6nHzAoztur87cNQRNYInVBBk7tjcCE83zVz3HrVDfxvumb2MPVMAVR8dmRIougX/DhbwrK8pwNEksnROriU884e9+PJDnaYXxwFS20=&cssminify=yes' type='text/css' media='all' />
|
||
<script id="jetpack_related-posts-js-extra">
|
||
var related_posts_js_options = {"post_heading":"h4"};
|
||
//# sourceURL=jetpack_related-posts-js-extra
|
||
</script>
|
||
<script id="wpcom-actionbar-placeholder-js-extra">
|
||
var actionbardata = {"siteID":"23592848","postID":"8832","siteURL":"https://blog.acolyer.org","xhrURL":"https://blog.acolyer.org/wp-admin/admin-ajax.php","nonce":"bc23e7bfe1","isLoggedIn":"","statusMessage":"","subsEmailDefault":"instantly","proxyScriptUrl":"https://s0.wp.com/wp-content/js/wpcom-proxy-request.js?m=1513050504i&ver=20211021","shortlink":"https://wp.me/p1AZzO-2is","i18n":{"followedText":"New posts from this site will now appear in your \u003Ca href=\"https://wordpress.com/reader\"\u003EReader\u003C/a\u003E","foldBar":"Collapse this bar","unfoldBar":"Expand this bar","shortLinkCopied":"Shortlink copied to clipboard."}};
|
||
//# sourceURL=wpcom-actionbar-placeholder-js-extra
|
||
</script>
|
||
<script id="jetpack-mu-wpcom-settings-js-before">
|
||
var JETPACK_MU_WPCOM_SETTINGS = {"assetsUrl":"https://s1.wp.com/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/moon/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/"};
|
||
//# sourceURL=jetpack-mu-wpcom-settings-js-before
|
||
</script>
|
||
<script crossorigin='anonymous' type='text/javascript' src='/_static/??/wp-content/mu-plugins/jetpack-plugin/moon/_inc/build/related-posts/related-posts.min.js,/wp-content/js/rlt-proxy.js?m=1788798821j'></script>
|
||
<script id="rlt-proxy-js-after">
|
||
rltInitialize( {"token":null,"iframeOrigins":["https:\/\/widgets.wp.com"]} );
|
||
//# sourceURL=rlt-proxy-js-after
|
||
</script>
|
||
<link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://adriancolyer.wordpress.com/xmlrpc.php?rsd" />
|
||
<meta name="generator" content="WordPress.com" />
|
||
<link rel="canonical" href="https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/" />
|
||
<link rel='shortlink' href='https://wp.me/p1AZzO-2is' />
|
||
<link rel="alternate" type="application/json+oembed" href="https://public-api.wordpress.com/oembed/?format=json&url=https%3A%2F%2Fblog.acolyer.org%2F2019%2F06%2F21%2Fwhat-bugs-cause-cloud-production-incidents%2F&for=wpcom-auto-discovery" /><link rel="alternate" type="application/xml+oembed" href="https://public-api.wordpress.com/oembed/?format=xml&url=https%3A%2F%2Fblog.acolyer.org%2F2019%2F06%2F21%2Fwhat-bugs-cause-cloud-production-incidents%2F&for=wpcom-auto-discovery" />
|
||
<!-- Jetpack Open Graph Tags -->
|
||
<meta property="og:type" content="article" />
|
||
<meta property="og:title" content="What bugs cause cloud production incidents?" />
|
||
<meta property="og:url" content="https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/" />
|
||
<meta property="og:description" content="What bugs cause production cloud incidents? Liu et al., HotOS’19 Last time out we looked at SLOs for cloud platforms, today we’re looking at what causes them to be broken! This is a stu…" />
|
||
<meta property="article:published_time" content="2019-06-21T05:00:00+00:00" />
|
||
<meta property="article:modified_time" content="2019-06-21T15:41:09+00:00" />
|
||
<meta property="og:site_name" content="the morning paper" />
|
||
<meta property="og:image" content="https://blog.acolyer.org/wp-content/uploads/2019/06/azure-incidents-fig-1.jpeg?w=480" />
|
||
<meta property="og:image:alt" content="" />
|
||
<meta property="og:locale" content="en_US" />
|
||
<meta property="article:publisher" content="https://www.facebook.com/WordPresscom" />
|
||
<meta name="twitter:text:title" content="What bugs cause cloud production incidents?" />
|
||
<meta name="twitter:image" content="https://blog.acolyer.org/wp-content/uploads/2019/06/azure-incidents-fig-1.jpeg?w=144" />
|
||
<meta name="twitter:card" content="summary" />
|
||
|
||
<!-- End Jetpack Open Graph Tags -->
|
||
<link rel="shortcut icon" type="image/x-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=32" sizes="16x16" />
|
||
<link rel="icon" type="image/x-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=32" sizes="16x16" />
|
||
<link rel="apple-touch-icon" href="https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=114" />
|
||
<link rel='openid.server' href='https://blog.acolyer.org/?openidserver=1' />
|
||
<link rel='openid.delegate' href='https://blog.acolyer.org/' />
|
||
<link rel="search" type="application/opensearchdescription+xml" href="https://blog.acolyer.org/osd.xml" title="the morning paper" />
|
||
<link rel="search" type="application/opensearchdescription+xml" href="https://s1.wp.com/opensearch.xml" title="WordPress.com" />
|
||
<meta name="theme-color" content="#ffffff" />
|
||
<meta name="description" content="What bugs cause production cloud incidents? Liu et al., HotOS'19 Last time out we looked at SLOs for cloud platforms, today we're looking at what causes them to be broken! This is a study of every high severity production incident at Microsoft Azure services over a span of six months, where the root cause of…" />
|
||
<style id="custom-background-css">
|
||
body.custom-background { background-color: #ffffff; }
|
||
</style>
|
||
<style type="text/css" id="custom-colors-css"> .widget a:hover,
|
||
.site-footer .widget a:hover {
|
||
color: #666;
|
||
}
|
||
.jetpack-social-navigation ul a:hover { color: #FFFFFF;}
|
||
body { background-color: #ffffff;}
|
||
.jetpack-social-navigation ul a:hover { background-color: #1c7c7c;}
|
||
.jetpack-recipe-title { border-color: #1c7c7c;}
|
||
.toggled .menu-toggle, .wp_widget_tag_cloud a:hover, .tags-links a:hover { border-color: #1B7878;}
|
||
.button:hover, .button:focus, .button:active, button:hover, button:focus, button:active, input[type="button"]:hover, input[type="button"]:focus, input[type="button"]:active, input[type="reset"]:hover, input[type="reset"]:focus, input[type="reset"]:active, input[type="submit"]:hover, input[type="submit"]:focus, input[type="submit"]:active, #infinite-handle span:hover { border-color: #1B7878;}
|
||
.button:hover, .button:focus, .button:active, button:hover, button:focus, button:active, input[type="button"]:hover, input[type="button"]:focus, input[type="button"]:active, input[type="reset"]:hover, input[type="reset"]:focus, input[type="reset"]:active, input[type="submit"]:hover, input[type="submit"]:focus, input[type="submit"]:active, #infinite-handle span:hover { color: #1B7878;}
|
||
.menu-toggle:hover #menu-icon, .toggled #menu-icon { fill: #1B7878;}
|
||
a, a:visited, .site-footer a:hover, .main-navigation li:hover > a, .current_page_item a, .current-menu-item a, .toggled .menu-toggle, .entry-footer a:hover, .entry-meta a:hover, .entry-title a:hover, .site-main .post-navigation .meta-nav, .site-main .paging-navigation .meta-nav, .comment-meta .comment-metadata a:hover, .required, .widget a, #wp-calendar a, .wp_widget_tag_cloud a:hover, #infinite-footer .container a:hover, #infinite-handle span, .tags-links a:hover, .current_page_item a, .current-menu-item a, .current_page_item a:visited, .current-menu-item a:visited { color: #1B7878;}
|
||
</style>
|
||
<link rel="stylesheet" id="custom-css-css" type="text/css" href="https://s2.wp.com/?custom-css=1&csblog=1AZzO&cscache=6&csrev=16" />
|
||
</head>
|
||
|
||
<body class="wp-singular post-template-default single single-post postid-8832 single-format-standard custom-background wp-embed-responsive wp-theme-pubpenscratch-2 customizer-styles-applied no-sidebar jetpack-reblog-enabled categories-hidden author-hidden custom-colors">
|
||
<div id="page" class="hfeed site">
|
||
<a class="skip-link screen-reader-text" href="#content">Skip to content</a>
|
||
<header id="masthead" class="site-header" role="banner">
|
||
<div class="site-branding">
|
||
<p class="site-title"><a href="https://blog.acolyer.org/" rel="home">the morning paper</a></p>
|
||
<p class="site-description">a random walk through Computer Science research, by Adrian Colyer</p>
|
||
</div>
|
||
|
||
<nav id="site-navigation" class="main-navigation" role="navigation">
|
||
<button class="menu-toggle">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><rect x="0" fill="none" width="16" height="16"/><g><path id="menu-icon" d="M0 14h16v-2H0v2zM0 2v2h16V2H0zm0 7h16V7H0v2z"/></g></svg>
|
||
Menu </button>
|
||
<div class="menu"><ul>
|
||
<li class="page_item page-item-2"><a href="https://blog.acolyer.org/about/">About</a></li>
|
||
<li class="page_item page-item-8716"><a href="https://blog.acolyer.org/archives/">Archives</a></li>
|
||
<li class="page_item page-item-5027"><a href="https://blog.acolyer.org/infoq-quarterly-review-editions/">InfoQ QR Editions</a></li>
|
||
<li class="page_item page-item-8713"><a href="https://blog.acolyer.org/search/">Search</a></li>
|
||
<li class="page_item page-item-406"><a href="https://blog.acolyer.org/email-subs/">Subscribe</a></li>
|
||
<li class="page_item page-item-8719"><a href="https://blog.acolyer.org/tags/">Tags</a></li>
|
||
<li class="page_item page-item-6991"><a href="https://blog.acolyer.org/privacy/">Privacy</a></li>
|
||
</ul></div>
|
||
</nav><!-- #site-navigation -->
|
||
</header><!-- #masthead -->
|
||
|
||
<div id="content" class="site-content">
|
||
|
||
<div id="primary" class="content-area">
|
||
<main id="main" class="site-main" role="main">
|
||
|
||
|
||
|
||
<article id="post-8832" class="post-8832 post type-post status-publish format-standard hentry category-uncategorized">
|
||
<header class="entry-header">
|
||
<h1 class="entry-title">What bugs cause cloud production incidents?</h1>
|
||
<div class="entry-meta">
|
||
<span class="posted-on"><a href="https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/" rel="bookmark"><time class="entry-date published" datetime="2019-06-21T06:00:00+01:00">June 21, 2019</time><time class="updated" datetime="2019-06-21T16:41:09+01:00">June 21, 2019</time></a></span><span class="byline"><span class="author vcard"><span class="sep"> ~ </span><a class="url fn n" href="https://blog.acolyer.org/author/adriancolyer/">adriancolyer</a></span></span> </div><!-- .entry-meta -->
|
||
</header><!-- .entry-header -->
|
||
|
||
<div class="entry-content">
|
||
|
||
<p class="wp-block-paragraph"><a href="https://people.cs.uchicago.edu/~shanlu/paper/hotos19_azure.pdf">What bugs cause production cloud incidents?</a> Liu et al., <em>HotOS’19</em></p>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">Last time out we looked at <a href="https://blog.acolyer.org/2019/06/19/nines-are-not-enough/">SLOs for cloud platforms</a>, today we’re looking at what causes them to be broken! This is a study of every high severity production incident at Microsoft Azure services over a span of six months, where the root cause of that incident was a software bug. In total, there were 112 such incidents over the period March – September 2018 (not all of them affecting external customers). Software bugs are the most common cause of incidents during this period, accounting for around 40% of all incidents (so we can infer there were around 280 incidents total in the pool).</p>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">The 112 incidents caused by software bugs are further broken down into categories, with <em>data-format</em> bugs, <em>fault-related</em> bugs, <em>timing</em> bugs, and <em>constant_value</em> bugs being the largest categories. Interestingly, outages caused by <em>configuration</em> errors represented only a small number of incidents in this study. This could be an artefact of that data set in some way, or it might be due to the tool chain that Microsoft uses:</p>
|
||
|
||
|
||
|
||
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
|
||
The types of bugs we observed in production are biased by the fact that Microsoft uses effective tools to mostly eliminate many types oft bugs before they can manifest in production, and hence our study includes zero or few of such bugs. For example, we observed only a small number of configuration bugs caused by mis-specification of configuration entries in configuration files, even though such bugs were reported to be common in other settings.
|
||
</p></blockquote>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">Most Azure code is written in .Net managed languages such as C#, reducing memory leak bugs. Tools like <a href="https://www.microsoft.com/en-us/research/project/chess-find-and-reproduce-heisenbugs-in-concurrent-programs/">CHESS</a> and <a href="https://www.microsoft.com/en-us/research/publication/a-randomized-scheduler-with-probabilistic-guarantees-of-finding-bugs/">PCT</a> are used to expose shared-memory concurrency bugs. TLA+ is used to model concurrent and distributed system protocols helping to eliminate high level design and semantic bugs. In addition, Azure’s Fault Analysis Service supports various types of fault injections during testing, such as node restart, data migration, and random faults. Microsoft is also using fuzz testing, cloud contract checking, and network configuration verification tools.</p>
|
||
|
||
|
||
|
||
<h3 class="wp-block-heading">Data formats</h3>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">Of the software bugs that survive all of this and end up causing high severity incidents, one of. the most common causes are data format change (21%).</p>
|
||
|
||
|
||
|
||
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
|
||
Different components of cloud services interact with each other through various types of “data”, including inter-process/node messages, persistent files, and so on. At the same time, cloud software goes through frequent updates. As a result, different software components in the cloud could hold conflicting assumptions about the format of certain data, leading to service incidents.
|
||
</p></blockquote>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">It looks like <a href="https://blog.acolyer.org/2019/06/05/data-validation-for-machine-learning/">data validation isn’t only useful in a machine learning context</a>!</p>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">All but one of the data format bugs involved multiple processes or nodes. In 40% of cases different parties assume different formats for shared files or database tables. For example, an upgrade has been deployed to a component ‘owning’ the table which changes the schema. The other 60% of cases are caused by a service changing the interface of its external message APIs. For example, a service that used to return 200 together with an empty list when no results were found changes to returning a 404, and breaks existing clients.</p>
|
||
|
||
|
||
|
||
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
|
||
The large scale, frequent updates, and long running nature of cloud services likely have facilitated the occurrence of these bugs.
|
||
</p></blockquote>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">I’d expect this class of bugs to also surface in microservices systems. Does that match your experience?</p>
|
||
|
||
|
||
|
||
<h3 class="wp-block-heading">Fault related</h3>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">Next up is an old chestnut: <a href="https://blog.acolyer.org/2016/10/06/simple-testing-can-prevent-most-critical-failures/">error and exception handling faults</a> (31%).</p>
|
||
|
||
|
||
|
||
<ul class="wp-block-list"><li>Components that fail and report an error that can’t be handled (e.g., missing exception handlers) (43% of this category)</li><li>Unresponsive components that hang and are not picked up by fault-detection mechanisms, leading to user-visible timeouts (29% of this category)</li><li>Silent corruption of data with no error detection code in place, leading to incorrect results returned to users (17% of this category)</li></ul>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">Exception / error handlers contribute to the incidents by either ignoring error reports (35%), over-reacting (35%), or containing bugs within the handlers themselves leading to infinite loops, timing issues, and so on (30%).</p>
|
||
|
||
|
||
|
||
<h3 class="wp-block-heading">Timing incidents</h3>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">13% of the incidents are related to timing, with only 14% of the timing incidents actually recorded as deadlocks. Compared to classic timing bugs racing between threads in the same process, here many of these bugs are about <em>race conditions between multiple nodes</em> and many of them are <em>racing on persistent data like cached firewall rules, configuration, database tables, and so on</em>.</p>
|
||
|
||
|
||
|
||
<h3 class="wp-block-heading">Constant-value setting incidents</h3>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">7% of all software bug incidents are caused by mistakes in constants: hard-code configuration, special purpose strings such as URLs, and enum-typed values.</p>
|
||
|
||
|
||
|
||
<h3 class="wp-block-heading">Bug resolution</h3>
|
||
|
||
|
||
|
||
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
|
||
Facing tight time pressure, more often than not, software bug incidents were resolved through a variety of <em>mitigation</em> techniques (56%) without patching the buggy code (44%), providing quick solutions to users and maximizing service availability.
|
||
</p></blockquote>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">(Mitigated incidents may well have been followed up later with code changes, but these aren’t recorded in the respective incident reports).</p>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">When mitigating there are three main types of mitigation uncovered:</p>
|
||
|
||
|
||
|
||
<ul class="wp-block-list"><li><strong>Code mitigation</strong> involves rolling back to an earlier version of the software</li><li><strong>Data mitigation</strong> involves manually restoring, cleaning up, or deleting data in a file, table, etc.</li><li><strong>Environment mitigation</strong> involves killing and restarting processes, migrating workloads, adding fail-over resources, etc..</li></ul>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">When mitigating, environment mitigations are the most common.</p>
|
||
|
||
|
||
|
||
<figure class="wp-block-image"><img src="https://blog.acolyer.org/wp-content/uploads/2019/06/azure-incidents-fig-1.jpeg?w=480" alt="" /></figure>
|
||
|
||
|
||
|
||
<p class="wp-block-paragraph">Although the kind of mitigate employed does vary based on root cause:</p>
|
||
|
||
|
||
|
||
<figure class="wp-block-image"><img src="https://blog.acolyer.org/wp-content/uploads/2019/06/azure-incidents-fig-2.jpeg?w=480" alt="" /></figure>
|
||
|
||
|
||
|
||
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow"><p>
|
||
Much recent work looked at how to automatically generate new patches. In comparison, automatically generating mitigation steps has not been well studied and is worth more attention in the future.
|
||
</p></blockquote>
|
||
<div id="jp-post-flair" class="sharedaddy sd-like-enabled sd-sharing-enabled"><div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-twitter"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-twitter-8832"
|
||
class="share-twitter sd-button share-icon"
|
||
href="https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/?share=twitter"
|
||
target="_blank"
|
||
aria-labelledby="sharing-twitter-8832"
|
||
>
|
||
<span id="sharing-twitter-8832" hidden>Share on X (Opens in new window)</span>
|
||
<span>X</span>
|
||
</a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-linkedin-8832"
|
||
class="share-linkedin sd-button share-icon"
|
||
href="https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/?share=linkedin"
|
||
target="_blank"
|
||
aria-labelledby="sharing-linkedin-8832"
|
||
>
|
||
<span id="sharing-linkedin-8832" hidden>Share on LinkedIn (Opens in new window)</span>
|
||
<span>LinkedIn</span>
|
||
</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-email-8832"
|
||
class="share-email sd-button share-icon"
|
||
href="mailto:?subject=%5BShared%20Post%5D%20What%20bugs%20cause%20cloud%20production%20incidents%3F&body=https%3A%2F%2Fblog.acolyer.org%2F2019%2F06%2F21%2Fwhat-bugs-cause-cloud-production-incidents%2F&share=email"
|
||
target="_blank"
|
||
aria-labelledby="sharing-email-8832"
|
||
data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you're having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="87e8ec426d" data-email-share-track-url="https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/?share=email">
|
||
<span id="sharing-email-8832" hidden>Email a link to a friend (Opens in new window)</span>
|
||
<span>Email</span>
|
||
</a></li><li class="share-print"><a rel="nofollow noopener noreferrer"
|
||
data-shared="sharing-print-8832"
|
||
class="share-print sd-button share-icon"
|
||
href="https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/#print?share=print"
|
||
target="_blank"
|
||
aria-labelledby="sharing-print-8832"
|
||
>
|
||
<span id="sharing-print-8832" hidden>Print (Opens in new window)</span>
|
||
<span>Print</span>
|
||
</a></li><li class="share-end"></li></ul></div></div></div><div class='sharedaddy sd-block sd-like jetpack-likes-widget-wrapper jetpack-likes-widget-unloaded' id='like-post-wrapper-23592848-8832-6aa2b69a415f9' data-src='//widgets.wp.com/likes/index.html?ver=20260910#blog_id=23592848&post_id=8832&origin=adriancolyer.wordpress.com&obj_id=23592848-8832-6aa2b69a415f9&domain=blog.acolyer.org' data-name='like-post-frame-23592848-8832-6aa2b69a415f9' data-title='Like or Reblog'><div class='likes-widget-placeholder post-likes-widget-placeholder' style='height: 55px;'><span class='button'><span>Like</span></span> <span class='loading'>Loading...</span></div><span class='sd-text-color'></span><a class='sd-link-color'></a></div>
|
||
<div id='jp-relatedposts' class='jp-relatedposts' >
|
||
<h3 class="jp-relatedposts-headline"><em>Related</em></h3>
|
||
</div></div> </div><!-- .entry-content -->
|
||
|
||
<footer class="entry-footer">
|
||
<span class="cat-links">
|
||
Posted in <a href="https://blog.acolyer.org/category/uncategorized/" rel="category tag">Uncategorized</a> </span>
|
||
</footer><!-- .entry-footer -->
|
||
|
||
</article><!-- #post-## -->
|
||
|
||
|
||
<nav class="navigation post-navigation" aria-label="Posts">
|
||
<h2 class="screen-reader-text">Post navigation</h2>
|
||
<div class="nav-links"><div class="nav-previous"><a href="https://blog.acolyer.org/2019/06/19/nines-are-not-enough/" rel="prev"><span class="meta-nav">‹ Previous</span>Nines are not enough: meaningful metrics for clouds</a></div><div class="nav-next"><a href="https://blog.acolyer.org/2019/06/24/fast-key-value-stores/" rel="next"><span class="meta-nav">Next ›</span>Fast key-value stores: an idea whose time has come and gone</a></div></div>
|
||
</nav>
|
||
|
||
<div id="comments" class="comments-area">
|
||
|
||
|
||
<h2 class="comments-title">
|
||
13 thoughts on “<span>What bugs cause cloud production incidents?</span>” </h2><!-- .comments-title -->
|
||
|
||
|
||
<ol class="comment-list">
|
||
<li id="comment-57617" class="pingback even thread-even depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://indian215720559.wordpress.com/2019/06/21/new-top-story-on-hacker-news-what-bugs-cause-cloud-production-incidents/" class="url" rel="ugc external nofollow">New top story on Hacker News: What bugs cause cloud production incidents? – Golden News</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57618" class="pingback odd alt thread-odd thread-alt depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://worldtimes6.wordpress.com/2019/06/21/new-top-story-on-hacker-news-what-bugs-cause-cloud-production-incidents/" class="url" rel="ugc external nofollow">New top story on Hacker News: What bugs cause cloud production incidents? – World Best News</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57620" class="pingback even thread-even depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://latestnewsdesign.wordpress.com/2019/06/21/new-top-story-on-hacker-news-what-bugs-cause-cloud-production-incidents/" class="url" rel="ugc external nofollow">New top story on Hacker News: What bugs cause cloud production incidents? – Latest news</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57621" class="pingback odd alt thread-odd thread-alt depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://hckrnews.wordpress.com/2019/06/21/new-top-story-on-hacker-news-what-bugs-cause-cloud-production-incidents/" class="url" rel="ugc external nofollow">New top story on Hacker News: What bugs cause cloud production incidents? – Hckr News</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57622" class="pingback even thread-even depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://myindianews1.wordpress.com/2019/06/21/what-bugs-cause-cloud-production-incidents/" class="url" rel="ugc external nofollow">What bugs cause cloud production incidents? – INDIA NEWS</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57623" class="pingback odd alt thread-odd thread-alt depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://newsblogie.wordpress.com/2019/06/21/new-top-story-on-hacker-news-what-bugs-cause-cloud-production-incidents/" class="url" rel="ugc external nofollow">New top story on Hacker News: What bugs cause cloud production incidents? – News about world</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57624" class="pingback even thread-even depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://outsidetheknow.wordpress.com/2019/06/21/new-top-story-on-hacker-news-what-bugs-cause-cloud-production-incidents/" class="url" rel="ugc external nofollow">New top story on Hacker News: What bugs cause cloud production incidents? – Outside The Know</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57628" class="pingback odd alt thread-odd thread-alt depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="http://popurls.com/pop/" class="url" rel="ugc external nofollow">=== popurls.com === popular today</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57631" class="pingback even thread-even depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://mytechblog.blogtown.co.nz/2019/06/what-bugs-cause-cloud-production-incidents/" class="url" rel="ugc external nofollow">What bugs cause cloud production incidents? | My Tech Blog</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57632" class="pingback odd alt thread-odd thread-alt depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="https://www.markbeazley.com/what-bugs-cause-cloud-production-incidents/" class="url" rel="ugc external nofollow">What bugs cause cloud production incidents? | Mark Beazley | MarkBeazley.Com</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57647" class="comment even thread-even depth-1">
|
||
<article id="div-comment-57647" class="comment-body">
|
||
<footer class="comment-meta">
|
||
<div class="comment-author vcard">
|
||
<img referrerpolicy="no-referrer" alt='Sandro Manke (闪山) (@2dbrain)'s avatar' src='https://i0.wp.com/pbs.twimg.com/profile_images/433928174955544576/yCXxwXaO_normal.jpeg?resize=60%2C60&ssl=1' srcset='https://i0.wp.com/pbs.twimg.com/profile_images/433928174955544576/yCXxwXaO_normal.jpeg?resize=60%2C60&ssl=1 1x, https://i0.wp.com/pbs.twimg.com/profile_images/433928174955544576/yCXxwXaO_normal.jpeg?resize=90%2C90&ssl=1 1.5x, https://i0.wp.com/pbs.twimg.com/profile_images/433928174955544576/yCXxwXaO_normal.jpeg?resize=120%2C120&ssl=1 2x, https://i0.wp.com/pbs.twimg.com/profile_images/433928174955544576/yCXxwXaO_normal.jpeg?resize=180%2C180&ssl=1 3x, https://i0.wp.com/pbs.twimg.com/profile_images/433928174955544576/yCXxwXaO_normal.jpeg?resize=240%2C240&ssl=1 4x' class='avatar avatar-60' height='60' width='60' decoding='async' /> <b class="fn"><a href="http://twitter.com/2dbrain" class="url" rel="ugc external nofollow">Sandro Manke (闪山) (@2dbrain)</a></b> <span class="says">says:</span> </div><!-- .comment-author -->
|
||
|
||
<div class="comment-metadata">
|
||
<a href="https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/#comment-57647"><time datetime="2019-06-21T11:47:47+01:00">June 21, 2019 at 11:47 am</time></a> </div><!-- .comment-metadata -->
|
||
|
||
</footer><!-- .comment-meta -->
|
||
|
||
<div class="comment-content">
|
||
<p>“I’d expect this class of bugs to also surface in microservices systems. Does that match your experience?”<br />
|
||
yes, very much exactly so :)</p>
|
||
</div><!-- .comment-content -->
|
||
|
||
</article><!-- .comment-body -->
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-57837" class="pingback odd alt thread-odd thread-alt depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="http://sreweekly.com/sre-weekly-issue-174/" class="url" rel="ugc external nofollow">SRE Weekly Issue #174 – SRE WEEKLY</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-58132" class="pingback even thread-even depth-1">
|
||
<div class="comment-body">
|
||
Pingback: <a href="http://blog.azure.moe/2019/06/27/azure-update-2019-06-27/" class="url" rel="ugc external nofollow">Azure Update (2019.06.27) | ブチザッキ</a> </div>
|
||
</li><!-- #comment-## -->
|
||
</ol><!-- .comment-list -->
|
||
|
||
|
||
|
||
<p class="no-comments">Comments are closed.</p>
|
||
|
||
|
||
</div><!-- #comments -->
|
||
|
||
|
||
</main><!-- #main -->
|
||
</div><!-- #primary -->
|
||
|
||
|
||
</div><!-- #content -->
|
||
|
||
<footer id="colophon" class="site-footer" role="contentinfo">
|
||
<div class="site-info-wrapper clear">
|
||
<div class="site-info">
|
||
<a href="https://wordpress.com/?ref=footer_blog" rel="nofollow">Blog at WordPress.com.</a>
|
||
|
||
</div><!-- .site-info -->
|
||
</div><!-- .site-info-wrapper -->
|
||
</footer><!-- #colophon -->
|
||
</div><!-- #page -->
|
||
|
||
<!-- -->
|
||
<script type="speculationrules">
|
||
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/files/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/pub/penscratch-2/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
|
||
</script>
|
||
<script id="grofiles-cards-js" src="//0.gravatar.com/js/hovercards/hovercards.min.js?ver=202637d1133547e8a1f2801609f51cef12522abe7f6647d869e9fbc974fe356a37e613"></script>
|
||
<script id="wpgroho-js-extra">
|
||
var WPGroHo = {"my_hash":""};
|
||
//# sourceURL=wpgroho-js-extra
|
||
</script>
|
||
<script crossorigin='anonymous' type='text/javascript' src='/wp-content/mu-plugins/gravatar-hovercards/wpgroho.js?m=1610363240i'></script>
|
||
|
||
<script>
|
||
// Initialize and attach hovercards to all gravatars
|
||
( function() {
|
||
function init() {
|
||
if ( typeof Gravatar === 'undefined' ) {
|
||
return;
|
||
}
|
||
|
||
if ( typeof Gravatar.init !== 'function' ) {
|
||
return;
|
||
}
|
||
|
||
Gravatar.profile_cb = function ( hash, id ) {
|
||
WPGroHo.syncProfileData( hash, id );
|
||
};
|
||
|
||
Gravatar.my_hash = WPGroHo.my_hash;
|
||
Gravatar.init(
|
||
'body',
|
||
'#wp-admin-bar-my-account',
|
||
{
|
||
i18n: {
|
||
'Edit your profile →': 'Edit your profile →',
|
||
'View profile →': 'View profile →',
|
||
'Contact': 'Contact',
|
||
'Send money': 'Send money',
|
||
'Sorry, we are unable to load this Gravatar profile.': 'Sorry, we are unable to load this Gravatar profile.',
|
||
'Gravatar not found.': 'Gravatar not found.',
|
||
'Too Many Requests.': 'Too Many Requests.',
|
||
'Internal Server Error.': 'Internal Server Error.',
|
||
'Is this you?': 'Is this you?',
|
||
'Claim your free profile.': 'Claim your free profile.',
|
||
'Email': 'Email',
|
||
'Home Phone': 'Home Phone',
|
||
'Work Phone': 'Work Phone',
|
||
'Cell Phone': 'Cell Phone',
|
||
'Contact Form': 'Contact Form',
|
||
'Calendar': 'Calendar',
|
||
},
|
||
}
|
||
);
|
||
}
|
||
|
||
if ( document.readyState !== 'loading' ) {
|
||
init();
|
||
} else {
|
||
document.addEventListener( 'DOMContentLoaded', init );
|
||
}
|
||
} )();
|
||
</script>
|
||
|
||
<div style="display:none">
|
||
<div class="grofile-hash-map-7fdf7a76981fdb742b6d3f203706ea79">
|
||
</div>
|
||
</div>
|
||
<div id="actionbar" dir="ltr" style="display: none;"
|
||
class="actnbr-pub-penscratch-2 actnbr-has-follow actnbr-has-actions">
|
||
<ul>
|
||
<li class="actnbr-btn actnbr-hidden">
|
||
<a class="actnbr-action actnbr-actn-reblog" href="">
|
||
<svg class="gridicon gridicons-reblog" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M22.086 9.914L20 7.828V18c0 1.105-.895 2-2 2h-7v-2h7V7.828l-2.086 2.086L14.5 8.5 19 4l4.5 4.5-1.414 1.414zM6 16.172V6h7V4H6c-1.105 0-2 .895-2 2v10.172l-2.086-2.086L.5 15.5 5 20l4.5-4.5-1.414-1.414L6 16.172z"/></g></svg><span>Reblog</span>
|
||
</a>
|
||
</li>
|
||
<li class="actnbr-btn actnbr-hidden">
|
||
<a class="actnbr-action actnbr-actn-follow " href="">
|
||
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
|
||
<span>Subscribe</span>
|
||
</a>
|
||
<a class="actnbr-action actnbr-actn-following no-display" href="">
|
||
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
|
||
<span>Subscribed</span>
|
||
</a>
|
||
<div class="actnbr-popover tip tip-top-left actnbr-notice" id="follow-bubble">
|
||
<div class="tip-arrow"></div>
|
||
<div class="tip-inner actnbr-follow-bubble">
|
||
<ul>
|
||
<li class="actnbr-sitename">
|
||
<a href="https://blog.acolyer.org">
|
||
<img loading='lazy' alt='' src='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png' srcset='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=75&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1.5x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=100&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 2x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=150&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 3x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=200&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 4x' class='avatar avatar-50' height='50' width='50' /> the morning paper </a>
|
||
</li>
|
||
<div class="actnbr-message no-display"></div>
|
||
<form method="post" action="https://subscribe.wordpress.com" accept-charset="utf-8" style="display: none;">
|
||
<div class="actnbr-follow-count">Join 3,125 other subscribers</div>
|
||
<div>
|
||
<input type="email" name="email" placeholder="Enter your email address" class="actnbr-email-field" aria-label="Enter your email address" />
|
||
</div>
|
||
<input type="hidden" name="action" value="subscribe" />
|
||
<input type="hidden" name="blog_id" value="23592848" />
|
||
<input type="hidden" name="source" value="https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/" />
|
||
<input type="hidden" name="sub-type" value="actionbar-follow" />
|
||
<input type="hidden" id="_wpnonce" name="_wpnonce" value="1314fd6265" /> <div class="actnbr-button-wrap">
|
||
<button type="submit" value="Sign me up">
|
||
Sign me up </button>
|
||
</div>
|
||
</form>
|
||
<li class="actnbr-login-nudge">
|
||
<div>
|
||
Already have a WordPress.com account? <a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fblog.acolyer.org%252F2019%252F06%252F21%252Fwhat-bugs-cause-cloud-production-incidents%252F">Log in now.</a> </div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</li>
|
||
<li class="actnbr-ellipsis actnbr-hidden">
|
||
<svg class="gridicon gridicons-ellipsis" height="24" width="24" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M7 12c0 1.104-.896 2-2 2s-2-.896-2-2 .896-2 2-2 2 .896 2 2zm12-2c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2zm-7 0c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2z"/></g></svg> <div class="actnbr-popover tip tip-top-left actnbr-more">
|
||
<div class="tip-arrow"></div>
|
||
<div class="tip-inner">
|
||
<ul>
|
||
<li class="actnbr-sitename">
|
||
<a href="https://blog.acolyer.org">
|
||
<img loading='lazy' alt='' src='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png' srcset='https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=50&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=75&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 1.5x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=100&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 2x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=150&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 3x, https://secure.gravatar.com/blavatar/a223d8685ecce4add34c037c09fa17d665b0daa6c6dc734f5984f129f7c5c2f8?s=200&d=https%3A%2F%2Fs2.wp.com%2Fi%2Flogo%2Fwpcom-gray-white.png 4x' class='avatar avatar-50' height='50' width='50' /> the morning paper </a>
|
||
</li>
|
||
<li class="actnbr-folded-follow">
|
||
<a class="actnbr-action actnbr-actn-follow " href="">
|
||
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
|
||
<span>Subscribe</span>
|
||
</a>
|
||
<a class="actnbr-action actnbr-actn-following no-display" href="">
|
||
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
|
||
<span>Subscribed</span>
|
||
</a>
|
||
</li>
|
||
<li class="actnbr-signup"><a href="https://wordpress.com/start/">Sign up</a></li>
|
||
<li class="actnbr-login"><a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fblog.acolyer.org%252F2019%252F06%252F21%252Fwhat-bugs-cause-cloud-production-incidents%252F">Log in</a></li>
|
||
<li class="actnbr-shortlink">
|
||
<a href="https://wp.me/p1AZzO-2is">
|
||
<span class="actnbr-shortlink__text">Copy shortlink</span>
|
||
<span class="actnbr-shortlink__icon"><svg class="gridicon gridicons-checkmark" height="16" width="16" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M9 19.414l-6.707-6.707 1.414-1.414L9 16.586 20.293 5.293l1.414 1.414"/></g></svg></span>
|
||
</a>
|
||
</li>
|
||
<li class="flb-report">
|
||
<a href="https://wordpress.com/abuse/?report_url=https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/" target="_blank" rel="noopener noreferrer">
|
||
Report this content </a>
|
||
</li>
|
||
<li class="actnbr-reader">
|
||
<a href="https://wordpress.com/reader/blogs/23592848/posts/8832">
|
||
View post in Reader </a>
|
||
</li>
|
||
<li class="actnbr-subs">
|
||
<a href="https://subscribe.wordpress.com/">Manage subscriptions</a>
|
||
</li>
|
||
<li class="actnbr-fold"><a href="">Collapse this bar</a></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
<script>
|
||
window.addEventListener( "DOMContentLoaded", function( event ) {
|
||
var link = document.createElement( "link" );
|
||
link.href = "/wp-content/mu-plugins/actionbar/actionbar.css?v=20250116";
|
||
link.type = "text/css";
|
||
link.rel = "stylesheet";
|
||
document.head.appendChild( link );
|
||
|
||
var script = document.createElement( "script" );
|
||
script.src = "/wp-content/mu-plugins/actionbar/actionbar.js?v=20250204";
|
||
document.body.appendChild( script );
|
||
} );
|
||
</script>
|
||
|
||
|
||
<script type="text/javascript">
|
||
window.WPCOM_sharing_counts = {"https://blog.acolyer.org/2019/06/21/what-bugs-cause-cloud-production-incidents/":8832};
|
||
</script>
|
||
|
||
<script type="text/javascript">
|
||
(function () {
|
||
var wpcom_reblog = {
|
||
source: 'toolbar',
|
||
|
||
toggle_reblog_box_flair: function (obj_id, post_id) {
|
||
|
||
// Go to site selector. This will redirect to their blog if they only have one.
|
||
const postEndpoint = `https://wordpress.com/post`;
|
||
|
||
// Ideally we would use the permalink here, but fortunately this will be replaced with the
|
||
// post permalink in the editor.
|
||
const originalURL = `${ document.location.href }?page_id=${ post_id }`;
|
||
|
||
const url =
|
||
postEndpoint +
|
||
'?url=' +
|
||
encodeURIComponent( originalURL ) +
|
||
'&is_post_share=true' +
|
||
'&v=5';
|
||
|
||
const redirect = function () {
|
||
if (
|
||
! window.open( url, '_blank' )
|
||
) {
|
||
location.href = url;
|
||
}
|
||
};
|
||
|
||
if ( /Firefox/.test( navigator.userAgent ) ) {
|
||
setTimeout( redirect, 0 );
|
||
} else {
|
||
redirect();
|
||
}
|
||
},
|
||
};
|
||
|
||
window.wpcom_reblog = wpcom_reblog;
|
||
})();
|
||
</script>
|
||
<script id="coblocks-loader-js-extra">
|
||
var wpcom_coblocks_js = {"coblocks_masonry_js":"https://s2.wp.com/wp-content/plugins/coblocks/2.18.1-simple-rev.4/dist/js/coblocks-masonry.min.js?m=1681832297i","coblocks_lightbox_js":"https://s2.wp.com/wp-content/plugins/coblocks/2.18.1-simple-rev.4/dist/js/coblocks-lightbox.min.js?m=1681832297i","jquery_core_js":"/wp-includes/js/jquery/jquery.min.js","jquery_migrate_js":"/wp-includes/js/jquery/jquery-migrate.min.js","masonry_js":"/wp-includes/js/masonry.min.js","imagesloaded_js":"/wp-includes/js/imagesloaded.min.js"};
|
||
var coblocksLigthboxData = {"closeLabel":"Close Gallery","leftLabel":"Previous","rightLabel":"Next"};
|
||
//# sourceURL=coblocks-loader-js-extra
|
||
</script>
|
||
<script id="sharing-js-js-extra">
|
||
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":"1"};
|
||
//# sourceURL=sharing-js-js-extra
|
||
</script>
|
||
<script crossorigin='anonymous' type='text/javascript' src='/_static/??-eJyVkN1uwjAMRl8I1ysaUC4QjzKliWndJE6oE37efmxiWsUdd/6kcz7LxmsGm6SQFIwVcqgDi2JgT4rnSpVGIy7Q3Ey6wgX8R9rUh2S94rppu6YF5ZgDwUyX5hMda8HpH4KQjHt0RZbXvjJSfKzMtcdMonY2xY6w/rHFXHgwhdM70jK/oannDIHFwynZqnDi26u9eNNEJRvrnxljSoJfLBb7ysGhjmYmZ5y7/44sw/PyYzy0u26333x02+30Ddt6jhE='></script>
|
||
<script id="sharing-js-js-after">
|
||
window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
|
||
window.WPCOM_sharing_popups[ "twitter" ] = "menubar=1,resizable=1,width=600,height=350";
|
||
window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
|
||
window.WPCOM_sharing_popups[ "linkedin" ] = "menubar=1,resizable=1,width=580,height=450";
|
||
//# sourceURL=sharing-js-js-after
|
||
</script>
|
||
<script id="wp-emoji-settings" type="application/json">
|
||
{"baseUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/72x72/","ext":".png","svgUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/svg/","svgExt":".svg","source":{"concatemoji":"/wp-includes/js/wp-emoji-release.min.js?m=1783090810i&ver=7.1.1-alpha-63555"}}
|
||
</script>
|
||
<script type="module">
|
||
/*! This file is auto-generated */
|
||
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
|
||
//# sourceURL=/wp-includes/js/wp-emoji-loader.min.js
|
||
</script>
|
||
<iframe src='https://widgets.wp.com/likes/master.html?ver=20260910#ver=20260910' scrolling='no' id='likes-master' name='likes-master' style='display:none;'></iframe>
|
||
<div id='likes-other-gravatars' role="dialog" aria-hidden="true" tabindex="-1"><div class="likes-text"><span>%d</span></div><ul class="wpl-avatars sd-like-gravatars"></ul></div>
|
||
<script src="//stats.wp.com/w.js?69" defer></script> <script type="text/javascript">
|
||
_tkq = window._tkq || [];
|
||
_stq = window._stq || [];
|
||
_tkq.push(['storeContext', {'blog_id':'23592848','blog_tz':'1','user_lang':'en','blog_lang':'en','user_id':'0'}]);
|
||
// Prevent sending pageview tracking from WP-Admin pages.
|
||
_stq.push(['view', {'blog':'23592848','v':'wpcom','tz':'1','user_id':'0','post':'8832','subd':'adriancolyer'}]);
|
||
_stq.push(['extra', {'crypt':'UE5tW3cvZGRDbzJxMWx6aVlydldVeXIvdUpWanJmOVg4Mys2XWxoN19+Lk5YJUp6QyVxQW02Q0lDM1lFMHBvZkEwd0d2fl9wNzlvdDY0ZnJpUCVPVy1fVltFNCZ4fFNoLkJhdXVsWkhsP0NDM3FBeCxqN21iQWc/eUVmbkgwWVRQaj9lPzlHdmpWdVhmWUNWV3ZsU2xFRjMlUFN6alk4UXB4RVp0OFQ3ZExuL0N1PUJZSEZTd35rcmNEQmg2OF83flNRLCxaQmZqdjNydGEzLC5FTVBjalZwRkhlc2hpQkg='}]);
|
||
_stq.push([ 'clickTrackerInit', '23592848', '8832' ]);
|
||
</script>
|
||
<noscript><img src="https://pixel.wp.com/b.gif?v=noscript" style="height:1px;width:1px;overflow:hidden;position:absolute;bottom:1px;" alt="" /></noscript>
|
||
<meta id="bilmur" property="bilmur:data" content="" data-provider="wordpress.com" data-service="simple" data-site-tz="Europe/London" data-custom-props="{"logged_in":"0","wptheme":"pub\/penscratch-2","wptheme_is_block":"0"}" >
|
||
<script defer src="/wp-content/js/bilmur.min.js?i=17&m=202637"></script>
|
||
</body>
|
||
</html>
|