Files
nexus/sreweekly/articles/413/04-justifying-resilience-work.html
2026-09-12 17:23:01 +08:00

348 lines
37 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="profile" href="http://gmpg.org/xfn/11">
<link rel="pingback" href="https://willgallego.com/xmlrpc.php">
<title>Justifying Resilience Work &#8211; Will Gallego</title>
<meta name='robots' content='max-image-preview:large' />
<style>img:is([sizes="auto" i], [sizes^="auto," i]) { contain-intrinsic-size: 3000px 1500px }</style>
<link rel='dns-prefetch' href='//fonts.googleapis.com' />
<link rel="alternate" type="application/rss+xml" title="Will Gallego &raquo; Feed" href="https://willgallego.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="Will Gallego &raquo; Comments Feed" href="https://willgallego.com/comments/feed/" />
<link rel="alternate" type="application/rss+xml" title="Will Gallego &raquo; Justifying Resilience Work Comments Feed" href="https://willgallego.com/2024/02/16/justifying-resilience-work/feed/" />
<script type="text/javascript">
/* <![CDATA[ */
window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/16.0.1\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/16.0.1\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/willgallego.com\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.8.8"}};
/*! This file is auto-generated */
!function(s,n){var o,i,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),a=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===a[t]})}function u(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);for(var n=e.getImageData(16,16,1,1),a=0;a<n.data.length;a++)if(0!==n.data[a])return!1;return!0}function f(e,t,n,a){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!a(e,"\ud83e\udedf")}return!1}function g(e,t,n,a){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):s.createElement("canvas"),o=r.getContext("2d",{willReadFrequently:!0}),i=(o.textBaseline="top",o.font="600 32px Arial",{});return e.forEach(function(e){i[e]=t(o,e,n,a)}),i}function t(e){var t=s.createElement("script");t.src=e,t.defer=!0,s.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",i=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){s.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+g.toString()+"("+[JSON.stringify(i),f.toString(),p.toString(),u.toString()].join(",")+"));",a=new Blob([e],{type:"text/javascript"}),r=new Worker(URL.createObjectURL(a),{name:"wpTestEmojiSupports"});return void(r.onmessage=function(e){c(n=e.data),r.terminate(),t(n)})}catch(e){}c(n=g(i,f,p,u))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings);
/* ]]> */
</script>
<style id='wp-emoji-styles-inline-css' type='text/css'>
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
</style>
<link rel='stylesheet' id='wp-block-library-css' href='https://willgallego.com/wp-includes/css/dist/block-library/style.min.css?ver=6.8.8' type='text/css' media='all' />
<style id='classic-theme-styles-inline-css' type='text/css'>
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
</style>
<style id='global-styles-inline-css' type='text/css'>
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}
:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}
:root :where(.wp-block-pullquote){font-size: 1.5em;line-height: 1.6;}
</style>
<link rel='stylesheet' id='arcade-basic-stylesheet-css' href='https://willgallego.com/wp-content/themes/arcade-basic/style.css?ver=6.8.8' type='text/css' media='all' />
<link rel='stylesheet' id='arcade-basic-fonts-css' href='//fonts.googleapis.com/css?family=Megrim|Raleway|Open+Sans:400,400italic,700,700italic' type='text/css' media='all' />
<link rel='stylesheet' id='font-awesome-css' href='https://willgallego.com/wp-content/themes/arcade-basic/library/css/font-awesome.css?ver=4.7.0' type='text/css' media='all' />
<script type="text/javascript" src="https://willgallego.com/wp-includes/js/jquery/jquery.min.js?ver=3.7.1" id="jquery-core-js"></script>
<script type="text/javascript" src="https://willgallego.com/wp-includes/js/jquery/jquery-migrate.min.js?ver=3.4.1" id="jquery-migrate-js"></script>
<link rel="https://api.w.org/" href="https://willgallego.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://willgallego.com/wp-json/wp/v2/posts/701" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://willgallego.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress 6.8.8" />
<link rel="canonical" href="https://willgallego.com/2024/02/16/justifying-resilience-work/" />
<link rel='shortlink' href='https://willgallego.com/?p=701' />
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://willgallego.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fwillgallego.com%2F2024%2F02%2F16%2Fjustifying-resilience-work%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://willgallego.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fwillgallego.com%2F2024%2F02%2F16%2Fjustifying-resilience-work%2F&#038;format=xml" />
<style>
.container { max-width: 1170px; }
</style>
<style type="text/css">.recentcomments a{display:inline !important;padding:0 !important;margin:0 !important;}</style></head>
<body class="wp-singular post-template-default single single-post postid-701 single-format-standard wp-theme-arcade-basic no-fittext basic">
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-106878020-1"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'UA-106878020-1');
</script>
<div id="page">
<header id="header">
<nav id="site-navigation" class="navbar navbar-inverse navbar-fixed-top" role="navigation">
<h3 class="sr-only">Main menu</h3>
<a class="sr-only" href="#primary" title="Skip to content">Skip to content</a>
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<div class="collapse navbar-collapse">
<ul id="menu-links" class="nav navbar-nav"><li id="menu-item-5" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-home menu-item-5"><a href="http://willgallego.com">Home</a></li>
<li id="menu-item-36" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-36"><a href="https://willgallego.com/about/">About</a></li>
<li id="menu-item-207" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-207"><a href="https://willgallego.com/talks/">Talks</a></li>
<li id="menu-item-7" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-7"><a target="_blank" href="https://twitter.com/wcgallego">Twitter</a></li>
<li id="menu-item-8" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-8"><a target="_blank" href="https://github.com/wcgallego">Github</a></li>
<li id="menu-item-651" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-651"><a target="_blank" rel="me" href="https://mastodon.social/@gallego">Mastodon</a></li>
</ul> </div>
</nav><!-- #site-navigation -->
<div class="title-card-wrapper">
<div class="title-card">
<div id="site-meta">
<div id="site-title">
<a href="https://willgallego.com" title="Will Gallego" rel="home">Will Gallego</a>
</div>
<i class="fa fa-gamepad"></i>
<div id="site-description">
Software Engineering, Blame Awareness, and gif with a soft ‘G’ </div>
<a href="#" id="more-site" class="btn btn-default btn-lg">See More</a>
</div>
<img src="https://willgallego.com/wp-content/uploads/2024/02/4304119797_77c035a8ca_c.jpg" alt="" class="header-img" /> </div>
</div>
</header>
<main>
<div class="container">
<div class="row">
<div id="primary" class="col-md-8 hfeed">
<article id="post-701" class="clearfix post-701 post type-post status-publish format-standard has-post-thumbnail hentry category-uncategorized xfolkentry">
<h1 class="entry-title">Justifying Resilience Work</h1>
<div class="entry-meta">
by <span class="vcard author"><span class="fn"><a href="https://willgallego.com/author/will/" title="Posts by Will Gallego" rel="author">Will Gallego</a></span></span>&nbsp;on&nbsp;<a href="https://willgallego.com/2024/02/16/justifying-resilience-work/" class="time"><time class="date published updated" datetime="2024-02-16">February 16, 2024</time></a>&nbsp;in&nbsp;<a href="https://willgallego.com/category/uncategorized/" rel="category tag">Uncategorized</a>&nbsp;&bull;&nbsp;<a href="https://willgallego.com/2024/02/16/justifying-resilience-work/#comments">2 Comments</a> </div>
<div class="entry-content description clearfix">
<p>How do we justify the need to make our organizations more resilient?<br><br><em>Quick Primer: in this context, a shorthand for resilience is adapting to unexpected changes in an environment in an attempt to return to stability. It&#8217;s not robustness, like an automatic fail-over of a system without human intervention (removing a server from a pool behind a load balancer for example).</em></p>
<p>Resilience Engineers often say we&#8217;re looking to improve the resilience of organizations because that&#8217;s what allows complex systems to function. That sounds great on paper &#8211; let&#8217;s be more resilient! How much do I budget, can I save some for a rainy day, and most importantly what SaaS should I buy my resilience from?</p>
<p>Specifically, you can&#8217;t have resilience without extra capacity. Resilience is by its nature imperfect in its efficiency. That&#8217;s not to say it is <em>inefficient</em> &#8211; quite the opposite. It has to be exploratory across typically untrodden or overlooked paths <em>for</em> optimizations (why else would we ignore an optimization?). This feeling of inefficiency is, then, the unknown cost ahead of time and how much investment is needed to reach &#8220;just enough resilience&#8221; to withstand the next storm.</p>
<p>It&#8217;s also unlike shoring up inefficiencies in optimizing code, deploy queue speeds, memory footprints, etc. We have no strong measurements on resilience for an organization, where it starts/stops, and how to fully justify the resilience after the fact. This is an excellent time to remind folks that I am not in sales.</p>
<p>I cite John Allspaw (because that sounds much classier than the truth of outright theft):</p>
<blockquote class="wp-block-quote is-layout-flow wp-block-quote-is-layout-flow">
<p>Resilience is the story of the incident that didn&#8217;t happen</p>
</blockquote>
<p>Which means to say that it&#8217;s hard if not impossible to measure what savings you&#8217;ll gain from that work. &#8220;Tell me when you&#8217;ll have an incident in the future and how much downtime you&#8217;ll save&#8221; (which is why we tend towards reactivity &#8211; feeling the pain and responding to it, notably with action items as the cure-all). We don&#8217;t know when we&#8217;ll need precise expertise or in what &#8220;quantity&#8221; if such a thing can be measured &#8211; but we know we&#8217;ll need it!</p>
<p>With this in mind, it&#8217;s challenging making the case that resilience is absolutely essential. What&#8217;s your ROI for resilience, your KPIs for understanding your system, the OKRs for a project on leveling up your team? Often your best attempt is to focus on robustness, which can feel more &#8220;solid&#8221; (&#8220;I&#8217;m going to expand the pool of servers before the influx of traffic this holiday weekend&#8221;) and potentially gain some expertise alongside.</p>
<p>Oddly enough, despite our efforts in bureaucracy, resilience <em>still happens</em>. &#8220;Great!&#8221; the unknowing manager says, &#8220;we&#8217;re getting it for free and we didn&#8217;t have to budget anything for it!&#8221;. Well, no. When your next incident strikes (which is unavoidable in even the best of systems), that will come with a cost &#8211; customer trust, lost revenue, public communications, and of course engineering time away from the planned roadmap.</p>
<p>We build capacity for our systems to withstand the unexpected when it relates to the robustness of our systems (the technical side of the socio-technical) &#8211; data redundancy, fail-over mechanisms, etc. &#8211; with a similar hand waving and prognosticating of doom and gloom, often because it&#8217;s easier to say &#8220;We&#8217;ll get 2x traffic, we need 3x spare capacity&#8221;, and running the numbers on that feels very tactile, concrete.</p>
<p>If we want to keep an emphasis on resilience, building up capacity beyond simply what our servers and networks can handle but instead focus on what <em>people</em> can handle, then we need to improve how we&#8217;re highlighting where to find that resilience. Find sources of building resilience in your organizations first to highlight the need. I have ideas on that I&#8217;ll cryptically leave for another post/conference talk.</p>
<p>Resilience exists and is ephemeral. It is amorphous but is not limitless. And most importantly of all, it&#8217;s is absolutely essential and very likely one of the places we invest in least.</p>
<p class="has-text-align-right"><em>Photo: <a href="https://flickr.com/photos/cynthiacloskey/4304119797">https://flickr.com/photos/cynthiacloskey/4304119797</a></em></p>
</div><!-- .entry-content -->
<footer class="clearfix">
</footer><!-- .entry -->
</article><!-- #post-701 -->
<div id="posts-pagination" class="clearfix">
<h3 class="sr-only">Post navigation</h3>
<div class="previous pull-left"><a href="https://willgallego.com/2024/01/10/blame-awareness-is-universal/" rel="prev">&larr; Blame Awareness is Universal</a></div>
<div class="next pull-right"><a href="https://willgallego.com/2024/03/24/srecon24-americas-recap/" rel="next">SREcon24 Americas Recap &rarr;</a></div>
</div><!-- #posts-pagination -->
<div id="comments" class="comments-area">
<h2 id="comments-title">
<i class="fa fa-comments"></i>&nbsp;
2 comments for &ldquo;<span>Justifying Resilience Work</span>&rdquo; </h2>
<ol class="commentlist">
<li id="comment-6994" class="pingback">
<div class="comment-body">
<i class="fa fa-paperclip"></i>
Pingback: <a href="https://sreweekly.com/sre-weekly-issue-1-2/" class="url" rel="ugc external nofollow">SRE Weekly Issue #1 &#8211; SRE WEEKLY</a> </div>
</li><!-- #comment-## -->
<li id="comment-6995" class="pingback">
<div class="comment-body">
<i class="fa fa-paperclip"></i>
Pingback: <a href="https://fde.cat/index.php/2024/02/26/sre-weekly-issue-413/" class="url" rel="ugc external nofollow">SRE Weekly Issue #413 &#8211; FDE</a> </div>
</li><!-- #comment-## -->
</ol><!-- .commentlist -->
<div id="respond" class="comment-respond">
<h3 id="reply-title" class="comment-reply-title">Leave a Reply <small><a rel="nofollow" id="cancel-comment-reply-link" href="/2024/02/16/justifying-resilience-work/#respond" style="display:none;">Cancel reply</a></small></h3><form action="https://willgallego.com/wp-comments-post.php" method="post" id="commentform" class="comment-form"><p class="comment-notes"><span id="email-notes">Your email address will not be published.</span> <span class="required-field-message">Required fields are marked <span class="required">*</span></span></p><p class="comment-form-comment"><label for="comment">Comment <span class="required">*</span></label> <textarea id="comment" name="comment" cols="45" rows="8" maxlength="65525" required></textarea></p><p class="comment-form-author"><label for="author">Name <span class="required">*</span></label> <input id="author" name="author" type="text" value="" size="30" maxlength="245" autocomplete="name" required /></p>
<p class="comment-form-email"><label for="email">Email <span class="required">*</span></label> <input id="email" name="email" type="email" value="" size="30" maxlength="100" aria-describedby="email-notes" autocomplete="email" required /></p>
<p class="comment-form-url"><label for="url">Website</label> <input id="url" name="url" type="url" value="" size="30" maxlength="200" autocomplete="url" /></p>
<p class="form-submit"><input name="submit" type="submit" id="submit" class="submit" value="Post Comment" /> <input type='hidden' name='comment_post_ID' value='701' id='comment_post_ID' />
<input type='hidden' name='comment_parent' id='comment_parent' value='0' />
</p><p style="display: none;"><input type="hidden" id="akismet_comment_nonce" name="akismet_comment_nonce" value="4b71c4d25d" /></p><p style="display: none !important;" class="akismet-fields-container" data-prefix="ak_"><label>&#916;<textarea name="ak_hp_textarea" cols="45" rows="8" maxlength="100"></textarea></label><input type="hidden" id="ak_js_1" name="ak_js" value="104"/><script>document.getElementById( "ak_js_1" ).setAttribute( "value", ( new Date() ).getTime() );</script></p></form> </div><!-- #respond -->
</div><!-- #comments .comments-area -->
</div>
<div id="secondary" class="col-md-4" role="complementary">
<aside id="recent-posts-4" class="widget widget_recent_entries">
<h3 class="widget-title">Recent Posts</h3>
<ul>
<li>
<a href="https://willgallego.com/2025/06/21/big-enough-to-fail/">Big Enough to Fail</a>
</li>
<li>
<a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/">An Incident Review of an Incident Review</a>
</li>
<li>
<a href="https://willgallego.com/2024/11/11/job-switching-and-titles/">Job Switching and Titles</a>
</li>
<li>
<a href="https://willgallego.com/2024/07/12/the-fridge-theory-of-blameless-organizations/">The Fridge Theory of Blameless Organizations</a>
</li>
</ul>
</aside><aside id="search-2" class="widget widget_search"><form role="search" method="get" class="search-form" action="https://willgallego.com/">
<label>
<span class="screen-reader-text">Search for:</span>
<input type="search" class="search-field" placeholder="Search &hellip;" value="" name="s" />
</label>
<input type="submit" class="search-submit" value="Search" />
</form></aside>
<aside id="recent-posts-2" class="widget widget_recent_entries">
<h3 class="widget-title">Recent Posts</h3>
<ul>
<li>
<a href="https://willgallego.com/2025/06/21/big-enough-to-fail/">Big Enough to Fail</a>
</li>
<li>
<a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/">An Incident Review of an Incident Review</a>
</li>
<li>
<a href="https://willgallego.com/2024/11/11/job-switching-and-titles/">Job Switching and Titles</a>
</li>
<li>
<a href="https://willgallego.com/2024/07/12/the-fridge-theory-of-blameless-organizations/">The Fridge Theory of Blameless Organizations</a>
</li>
<li>
<a href="https://willgallego.com/2024/06/21/how-many-maybes-until-empathy/">How Many Maybe&#8217;s until Empathy?</a>
</li>
</ul>
</aside><aside id="recent-comments-2" class="widget widget_recent_comments"><h3 class="widget-title">Recent Comments</h3><ul id="recentcomments"><li class="recentcomments"><span class="comment-author-link"><a href="https://www.arnoldit.com/wordpress/2025/12/04/titanic-talk-this-ship-will-not-fail/" class="url" rel="ugc external nofollow">Titanic Talk: This Ship Will Not Fail : Stephen E. Arnold @ Beyond Search</a></span> on <a href="https://willgallego.com/2025/06/21/big-enough-to-fail/#comment-37339">Big Enough to Fail</a></li><li class="recentcomments"><span class="comment-author-link"><a href="https://sreweekly.com/sre-weekly-issue-484/" class="url" rel="ugc external nofollow">SRE Weekly Issue #484 &#8211; SRE WEEKLY</a></span> on <a href="https://willgallego.com/2025/06/21/big-enough-to-fail/#comment-25648">Big Enough to Fail</a></li><li class="recentcomments"><span class="comment-author-link">Jacob Rose</span> on <a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/#comment-17389">An Incident Review of an Incident Review</a></li><li class="recentcomments"><span class="comment-author-link">Leonardo Camara</span> on <a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/#comment-16366">An Incident Review of an Incident Review</a></li><li class="recentcomments"><span class="comment-author-link"><a href="https://sreweekly.com/sre-weekly-issue-460/" class="url" rel="ugc external nofollow">SRE Weekly Issue #460 &#8211; SRE WEEKLY</a></span> on <a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/#comment-16284">An Incident Review of an Incident Review</a></li></ul></aside><aside id="archives-2" class="widget widget_archive"><h3 class="widget-title">Archives</h3>
<ul>
<li><a href='https://willgallego.com/2025/06/'>June 2025</a></li>
<li><a href='https://willgallego.com/2025/01/'>January 2025</a></li>
<li><a href='https://willgallego.com/2024/11/'>November 2024</a></li>
<li><a href='https://willgallego.com/2024/07/'>July 2024</a></li>
<li><a href='https://willgallego.com/2024/06/'>June 2024</a></li>
<li><a href='https://willgallego.com/2024/03/'>March 2024</a></li>
<li><a href='https://willgallego.com/2024/02/'>February 2024</a></li>
<li><a href='https://willgallego.com/2024/01/'>January 2024</a></li>
<li><a href='https://willgallego.com/2023/10/'>October 2023</a></li>
<li><a href='https://willgallego.com/2023/04/'>April 2023</a></li>
<li><a href='https://willgallego.com/2023/01/'>January 2023</a></li>
<li><a href='https://willgallego.com/2022/12/'>December 2022</a></li>
<li><a href='https://willgallego.com/2022/06/'>June 2022</a></li>
<li><a href='https://willgallego.com/2022/02/'>February 2022</a></li>
<li><a href='https://willgallego.com/2022/01/'>January 2022</a></li>
<li><a href='https://willgallego.com/2021/08/'>August 2021</a></li>
<li><a href='https://willgallego.com/2021/05/'>May 2021</a></li>
<li><a href='https://willgallego.com/2021/02/'>February 2021</a></li>
<li><a href='https://willgallego.com/2021/01/'>January 2021</a></li>
<li><a href='https://willgallego.com/2020/09/'>September 2020</a></li>
<li><a href='https://willgallego.com/2020/06/'>June 2020</a></li>
<li><a href='https://willgallego.com/2020/05/'>May 2020</a></li>
<li><a href='https://willgallego.com/2020/02/'>February 2020</a></li>
<li><a href='https://willgallego.com/2019/12/'>December 2019</a></li>
<li><a href='https://willgallego.com/2019/08/'>August 2019</a></li>
<li><a href='https://willgallego.com/2019/07/'>July 2019</a></li>
<li><a href='https://willgallego.com/2019/04/'>April 2019</a></li>
<li><a href='https://willgallego.com/2019/02/'>February 2019</a></li>
<li><a href='https://willgallego.com/2018/12/'>December 2018</a></li>
<li><a href='https://willgallego.com/2018/08/'>August 2018</a></li>
<li><a href='https://willgallego.com/2018/07/'>July 2018</a></li>
<li><a href='https://willgallego.com/2018/04/'>April 2018</a></li>
<li><a href='https://willgallego.com/2018/02/'>February 2018</a></li>
<li><a href='https://willgallego.com/2018/01/'>January 2018</a></li>
<li><a href='https://willgallego.com/2017/11/'>November 2017</a></li>
<li><a href='https://willgallego.com/2017/10/'>October 2017</a></li>
<li><a href='https://willgallego.com/2017/09/'>September 2017</a></li>
</ul>
</aside> </div><!-- #secondary.widget-area --> </div>
</div>
</main><!-- main -->
<footer id="footer" role="contentinfo">
<div id="footer-content" class="container">
<div class="row">
<div class="copyright col-lg-12">
<span class="pull-left">Copyright &copy; 2026 <a href="https://willgallego.com">Will Gallego</a>. All Rights Reserved.</span>
<span class="credit-link pull-right"><i class="fa fa-leaf"></i>The Arcade Basic Theme by <a href="https://themes.bavotasan.com/themes/arcade">bavotasan.com</a>.</span>
</div><!-- .col-lg-12 -->
</div><!-- .row -->
</div><!-- #footer-content.container -->
</footer><!-- #footer -->
</div><!-- #page -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"\/*"},{"not":{"href_matches":["\/wp-*.php","\/wp-admin\/*","\/wp-content\/uploads\/*","\/wp-content\/*","\/wp-content\/plugins\/*","\/wp-content\/themes\/arcade-basic\/*","\/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script type="text/javascript" src="https://willgallego.com/wp-includes/js/comment-reply.min.js?ver=6.8.8" id="comment-reply-js" async="async" data-wp-strategy="async"></script>
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/bootstrap.min.js?ver=3.0.3" id="bootstrap-js"></script>
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/fillsize.js?ver=6.8.8" id="fillsize-js"></script>
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/jquery.arctext.js?ver=6.8.8" id="arctext-js"></script>
<script type="text/javascript" id="arcade-basic-script-js-extra">
/* <![CDATA[ */
var arcade_basic_vars = {"arc":"800","fittext":""};
/* ]]> */
</script>
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/theme.js?ver=6.8.8" id="arcade-basic-script-js"></script>
<script defer type="text/javascript" src="https://willgallego.com/wp-content/plugins/akismet/_inc/akismet-frontend.js?ver=1758810644" id="akismet-frontend-js"></script>
</body>
</html>
<!-- Page cached by LiteSpeed Cache 7.5.0.1 on 2026-09-08 23:25:45 -->