384 lines
41 KiB
HTML
384 lines
41 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en-US">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<link rel="profile" href="http://gmpg.org/xfn/11">
|
||
<link rel="pingback" href="https://willgallego.com/xmlrpc.php">
|
||
<title>Big Enough to Fail – Will Gallego</title>
|
||
<meta name='robots' content='max-image-preview:large' />
|
||
<style>img:is([sizes="auto" i], [sizes^="auto," i]) { contain-intrinsic-size: 3000px 1500px }</style>
|
||
<link rel='dns-prefetch' href='//fonts.googleapis.com' />
|
||
<link rel="alternate" type="application/rss+xml" title="Will Gallego » Feed" href="https://willgallego.com/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="Will Gallego » Comments Feed" href="https://willgallego.com/comments/feed/" />
|
||
<link rel="alternate" type="application/rss+xml" title="Will Gallego » Big Enough to Fail Comments Feed" href="https://willgallego.com/2025/06/21/big-enough-to-fail/feed/" />
|
||
<script type="text/javascript">
|
||
/* <![CDATA[ */
|
||
window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/16.0.1\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/16.0.1\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/willgallego.com\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.8.8"}};
|
||
/*! This file is auto-generated */
|
||
!function(s,n){var o,i,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),a=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===a[t]})}function u(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);for(var n=e.getImageData(16,16,1,1),a=0;a<n.data.length;a++)if(0!==n.data[a])return!1;return!0}function f(e,t,n,a){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!a(e,"\ud83e\udedf")}return!1}function g(e,t,n,a){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):s.createElement("canvas"),o=r.getContext("2d",{willReadFrequently:!0}),i=(o.textBaseline="top",o.font="600 32px Arial",{});return e.forEach(function(e){i[e]=t(o,e,n,a)}),i}function t(e){var t=s.createElement("script");t.src=e,t.defer=!0,s.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",i=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){s.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+g.toString()+"("+[JSON.stringify(i),f.toString(),p.toString(),u.toString()].join(",")+"));",a=new Blob([e],{type:"text/javascript"}),r=new Worker(URL.createObjectURL(a),{name:"wpTestEmojiSupports"});return void(r.onmessage=function(e){c(n=e.data),r.terminate(),t(n)})}catch(e){}c(n=g(i,f,p,u))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings);
|
||
/* ]]> */
|
||
</script>
|
||
|
||
<style id='wp-emoji-styles-inline-css' type='text/css'>
|
||
|
||
img.wp-smiley, img.emoji {
|
||
display: inline !important;
|
||
border: none !important;
|
||
box-shadow: none !important;
|
||
height: 1em !important;
|
||
width: 1em !important;
|
||
margin: 0 0.07em !important;
|
||
vertical-align: -0.1em !important;
|
||
background: none !important;
|
||
padding: 0 !important;
|
||
}
|
||
</style>
|
||
<link rel='stylesheet' id='wp-block-library-css' href='https://willgallego.com/wp-includes/css/dist/block-library/style.min.css?ver=6.8.8' type='text/css' media='all' />
|
||
<style id='classic-theme-styles-inline-css' type='text/css'>
|
||
/*! This file is auto-generated */
|
||
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
|
||
</style>
|
||
<style id='global-styles-inline-css' type='text/css'>
|
||
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
|
||
:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}
|
||
:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}
|
||
:root :where(.wp-block-pullquote){font-size: 1.5em;line-height: 1.6;}
|
||
</style>
|
||
<link rel='stylesheet' id='arcade-basic-stylesheet-css' href='https://willgallego.com/wp-content/themes/arcade-basic/style.css?ver=6.8.8' type='text/css' media='all' />
|
||
<link rel='stylesheet' id='arcade-basic-fonts-css' href='//fonts.googleapis.com/css?family=Megrim|Raleway|Open+Sans:400,400italic,700,700italic' type='text/css' media='all' />
|
||
<link rel='stylesheet' id='font-awesome-css' href='https://willgallego.com/wp-content/themes/arcade-basic/library/css/font-awesome.css?ver=4.7.0' type='text/css' media='all' />
|
||
<script type="text/javascript" src="https://willgallego.com/wp-includes/js/jquery/jquery.min.js?ver=3.7.1" id="jquery-core-js"></script>
|
||
<script type="text/javascript" src="https://willgallego.com/wp-includes/js/jquery/jquery-migrate.min.js?ver=3.4.1" id="jquery-migrate-js"></script>
|
||
<link rel="https://api.w.org/" href="https://willgallego.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://willgallego.com/wp-json/wp/v2/posts/800" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://willgallego.com/xmlrpc.php?rsd" />
|
||
<meta name="generator" content="WordPress 6.8.8" />
|
||
<link rel="canonical" href="https://willgallego.com/2025/06/21/big-enough-to-fail/" />
|
||
<link rel='shortlink' href='https://willgallego.com/?p=800' />
|
||
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://willgallego.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fwillgallego.com%2F2025%2F06%2F21%2Fbig-enough-to-fail%2F" />
|
||
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://willgallego.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fwillgallego.com%2F2025%2F06%2F21%2Fbig-enough-to-fail%2F&format=xml" />
|
||
<style>
|
||
.container { max-width: 1170px; }
|
||
</style>
|
||
<style type="text/css">.recentcomments a{display:inline !important;padding:0 !important;margin:0 !important;}</style></head>
|
||
<body class="wp-singular post-template-default single single-post postid-800 single-format-standard wp-theme-arcade-basic no-fittext basic">
|
||
|
||
<!-- Global site tag (gtag.js) - Google Analytics -->
|
||
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-106878020-1"></script>
|
||
<script>
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
gtag('js', new Date());
|
||
|
||
gtag('config', 'UA-106878020-1');
|
||
</script>
|
||
|
||
|
||
|
||
<div id="page">
|
||
|
||
<header id="header">
|
||
<nav id="site-navigation" class="navbar navbar-inverse navbar-fixed-top" role="navigation">
|
||
<h3 class="sr-only">Main menu</h3>
|
||
<a class="sr-only" href="#primary" title="Skip to content">Skip to content</a>
|
||
|
||
<div class="navbar-header">
|
||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
</div>
|
||
|
||
<div class="collapse navbar-collapse">
|
||
<ul id="menu-links" class="nav navbar-nav"><li id="menu-item-5" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-home menu-item-5"><a href="http://willgallego.com">Home</a></li>
|
||
<li id="menu-item-36" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-36"><a href="https://willgallego.com/about/">About</a></li>
|
||
<li id="menu-item-207" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-207"><a href="https://willgallego.com/talks/">Talks</a></li>
|
||
<li id="menu-item-7" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-7"><a target="_blank" href="https://twitter.com/wcgallego">Twitter</a></li>
|
||
<li id="menu-item-8" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-8"><a target="_blank" href="https://github.com/wcgallego">Github</a></li>
|
||
<li id="menu-item-651" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-651"><a target="_blank" rel="me" href="https://mastodon.social/@gallego">Mastodon</a></li>
|
||
</ul> </div>
|
||
</nav><!-- #site-navigation -->
|
||
|
||
<div class="title-card-wrapper">
|
||
<div class="title-card">
|
||
<div id="site-meta">
|
||
<div id="site-title">
|
||
<a href="https://willgallego.com" title="Will Gallego" rel="home">Will Gallego</a>
|
||
</div>
|
||
|
||
<i class="fa fa-gamepad"></i>
|
||
|
||
<div id="site-description">
|
||
Software Engineering, Blame Awareness, and gif with a soft ‘G’ </div>
|
||
<a href="#" id="more-site" class="btn btn-default btn-lg">See More</a>
|
||
</div>
|
||
|
||
<img src="https://willgallego.com/wp-content/uploads/2025/06/bridge.jpg" alt="" class="header-img" /> </div>
|
||
</div>
|
||
|
||
</header>
|
||
|
||
<main>
|
||
<div class="container">
|
||
<div class="row">
|
||
<div id="primary" class="col-md-8 hfeed">
|
||
|
||
<article id="post-800" class="clearfix post-800 post type-post status-publish format-standard has-post-thumbnail hentry category-tech tag-resilience-engineering xfolkentry">
|
||
|
||
<h1 class="entry-title">Big Enough to Fail</h1>
|
||
<div class="entry-meta">
|
||
by <span class="vcard author"><span class="fn"><a href="https://willgallego.com/author/will/" title="Posts by Will Gallego" rel="author">Will Gallego</a></span></span> on <a href="https://willgallego.com/2025/06/21/big-enough-to-fail/" class="time"><time class="date published updated" datetime="2025-06-21">June 21, 2025</time></a> in <a href="https://willgallego.com/category/tech/" rel="category tag">Tech</a> • <a href="https://willgallego.com/2025/06/21/big-enough-to-fail/#comments">2 Comments</a> </div>
|
||
|
||
<div class="entry-content description clearfix">
|
||
|
||
<p>I’ve been playing with an idea based mostly on anecdotal evidence: <strong>At some point, the external dependencies which our systems rely on become so tightly coupled, large, and fundamental that should those foundations inevitably fail, that blame can actually go <em>down</em> in response to an incident.</strong></p>
|
||
|
||
|
||
|
||
<p>A blanket statement applicable to any system feels egregious, but I’m confident enough to apply this to the sociotechnical systems of software development. This might seem odd – large failure modes are typically frowned upon, wide reaching and deeply felt with costs pushed to customers. Why would we give these highly critical services a pass?</p>
|
||
|
||
|
||
|
||
<h2 class="wp-block-heading">The unbelievable happens every day</h2>
|
||
|
||
|
||
|
||
<p>Take the recent <a href="https://status.cloud.google.com/incidents/ow5i3PPK96RduMcb1SsW" target="_blank" rel="noreferrer noopener">GCP failure</a>: A major provider of internet services has a major outage, one that is in all likelihood recoverable but will take some time. It even coincides with an <a href="https://blog.cloudflare.com/es-es/cloudflare-service-outage-june-12-2025/">outage at Cloudflare</a> (the rumor mill saying the third party vendor mentioned was GCP). Who knows how many working hours were devoted to localized investigations, the money lost from sales and time invested, with customers of these services that provide further services downstream to others? Yet when something as widespread as a cloud provider has issues, when someone as big as <em>Alphabet</em> has a disruption, it somehow becomes much more understandable than a mid-tier SaaS having an outage.</p>
|
||
|
||
|
||
|
||
<p><a href="https://help.salesforce.com/s/articleView?id=005036674&type=1" target="_blank" rel="noreferrer noopener">Salesforce had a major outage</a> across many major services as well recently. With the proliferation of Slack being near omnipresent in organizations, only one of many products that was impacted, whenever it goes down it has a large ripple effect across the industry in how we communicate. Even if that isn’t a service within your org, it’s highly likely that you’re reliant upon another company that has at least one Salesforce product as a dependency.</p>
|
||
|
||
|
||
|
||
<p>There’s likely some grumbling online and internally, sure, but are we shopping around for replacements? Often no, with exceptions of course – see <a href="https://www.reuters.com/business/media-telecom/disney-stop-using-salesforce-owned-slack-after-hack-exposed-company-data-wsj-2024-09-19/" target="_blank" rel="noreferrer noopener">Disney ditches Slack</a>. I don’t think it’s some magic threshold where no wrong can be done to dislodge customers. On the whole, though, we’re pretty forgiving of big tech when things go wrong, despite the cost downstream.</p>
|
||
|
||
|
||
|
||
<h2 class="wp-block-heading">Rationalizing choices at scale</h2>
|
||
|
||
|
||
|
||
<p>So why are we more tolerant of this level of outage to such core pieces of our infrastructure? I have a few hunches that could use more scientific rigor:</p>
|
||
|
||
|
||
|
||
<ul class="wp-block-list">
|
||
<li><strong>It’s so exceptional (or feels that way).</strong> This is less so about frequency but that when a company becomes so big you just assume they’re impervious to failure, a shock and awe to the impossible.</li>
|
||
|
||
|
||
|
||
<li><strong>The lack of choices in services informs your response.</strong> Are there other providers? Sure, but with the continuous consolidation of businesses, we have fewer options every day.</li>
|
||
|
||
|
||
|
||
<li><strong>You’re locked in on your choices.</strong> Are you going to knock on Google’s door and complain, take three years to move out of one virtual data center and into another, while retraining your staff, updating your internal documents, and updating your code? No, you’re likely not.</li>
|
||
|
||
|
||
|
||
<li><strong>Failover is costly.</strong> Similarly, those at the sharp end know that the level of effort in building failover for something like this is frequently impractical. It would cost too much to set up, to maintain as developers, it would remove effort that could be put towards new features, and the financial cost backing that might be considered infeasible.</li>
|
||
|
||
|
||
|
||
<li><strong>The brittleness is everywhere.</strong> The level of complexity and the highly coupled nature of interconnected services means we’ve become brittle to failures. Doubly so when those services are the underpinnings of what we build on. “The internet is down today” as the saying goes, despite the internet having no principle nucleus. This is considered acceptable.</li>
|
||
|
||
|
||
|
||
<li><strong>We’re all in it together.</strong> When a service as large as these goes down, there’s a good chance we’re seeing so many failures in so many places that it becomes reasonable to also be down. Your competitors are likely down, your customers might be – there might be too much failure to go around to cast it in any one direction.</li>
|
||
</ul>
|
||
|
||
|
||
|
||
<p>This may also be predicated upon the misconception that if you’re of sufficient size you should never have downtime, the assumption that failure is always a choice. Being such a fundamental building block of other downstream services, you should prepare for every eventuality and therefore never fail. <strong>Even Alphabet, Amazon, Meta, Apple, and others of a similar size have finite resources</strong>. There isn’t enough money, time, or people to make sure a company never has failures. Even well before that, companies are more likely to underestimate future failures, under prepare, and assume pass performance will indicate future success. We’re currently seeing the unbelievable, the farcical idea that <em>anyone</em> can be a victim of failure, and that shocking revelation has a strange calming effect when blame is to be cast.</p>
|
||
|
||
|
||
|
||
<p>And maybe saying it’s blame free is incorrect – there are certainly individuals who will still happily point fingers in such a case, because it’s free and makes them feel better. Maybe they squeeze a better deal out of an upcoming contract because of it. “Well you chose the wrong service”, which is very convenient in that both the assumption other services have never/will never fail, the costs associated with them are neglected, and this was something foreseeable. The inconvenience of an outage can be very convenient to localized goals.</p>
|
||
|
||
|
||
|
||
<h2 class="wp-block-heading">How do we create resilience with this in mind?</h2>
|
||
|
||
|
||
|
||
<p>One of the objections to Resilience Engineering is often “this all sounds fine, but what do I <em>do</em> with this information?”. It’s the challenge with invisible work and glue work, the idea of preventing and preparing doesn’t mean it’s immediately present – unless you go looking for it.</p>
|
||
|
||
|
||
|
||
<p>It’s good to think about what parallels we can draw from other systems and environments. If you have a power outage, you compensate by keeping refrigerators closed more often or, should desperation befall you, start making your way through the ice cream in the freezer. A burst pipe or water contamination may force the issue of a boil water advisory. Severe flooding can force a response of trivial issues like tying down or bringing in outdoor furniture all the way to a full on evacuation. Much like living free from any environmental disasters, it may be impractical or impossible to avoid. Atypical failures warrant some level of prep to help support the adaptation.</p>
|
||
|
||
|
||
|
||
<p>If we’re giving ourselves an opportunity to look past simplistic failure modes (“Pick a service that never goes down”), that’s a chance to consider options elsewhere in our ongoing work towards reliability. A core part of Resilience Engineering is reframing how we look at incidents – their ability to highlight gaps in the system and produce insights. For example, incident response can be more than fixing an issue. Are alerts meaningful, indicating failure modes that are actionable? Do folks feel confident adding to a status page? When we look for error modes, do we flail about with uncertainty while we gather information, or is there systemic tooling that helps gather that information? Think about all the questions you can ask yourself about your system in response to the extraordinary.</p>
|
||
|
||
|
||
|
||
<p>This could be a whole talk, but it’s clear that “well, it’s not <em>our</em> bug” isn’t a sufficient answer, even if we’re not pointing fingers. If we’re giving ourselves the benefit of being blame free, or however we frame it, we can also give ourselves the chance to put time and energy to parts of our organization often overlooked. And maybe given some of those smaller companies a break too. After all, no one’s perfect.<br></p>
|
||
|
||
|
||
|
||
<p class="has-text-align-right"><em>Photo: <a href="https://www.geograph.org.uk/photo/4966439">https://www.geograph.org.uk/photo/4966439</a></em></p>
|
||
</div><!-- .entry-content -->
|
||
<footer class="clearfix">
|
||
<p class="tags"><i class="fa fa-tags"></i> <span>Tags:</span><a href="https://willgallego.com/tag/resilience-engineering/" rel="tag">Resilience Engineering</a></p> </footer><!-- .entry -->
|
||
</article><!-- #post-800 -->
|
||
<div id="posts-pagination" class="clearfix">
|
||
<h3 class="sr-only">Post navigation</h3>
|
||
<div class="previous pull-left"><a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/" rel="prev">← An Incident Review of an Incident Review</a></div>
|
||
<div class="next pull-right"></div>
|
||
</div><!-- #posts-pagination -->
|
||
|
||
|
||
<div id="comments" class="comments-area">
|
||
|
||
<h2 id="comments-title">
|
||
<i class="fa fa-comments"></i>
|
||
2 comments for “<span>Big Enough to Fail</span>” </h2>
|
||
|
||
<ol class="commentlist">
|
||
<li id="comment-25648" class="pingback">
|
||
<div class="comment-body">
|
||
<i class="fa fa-paperclip"></i>
|
||
Pingback: <a href="https://sreweekly.com/sre-weekly-issue-484/" class="url" rel="ugc external nofollow">SRE Weekly Issue #484 – SRE WEEKLY</a> </div>
|
||
</li><!-- #comment-## -->
|
||
<li id="comment-37339" class="pingback">
|
||
<div class="comment-body">
|
||
<i class="fa fa-paperclip"></i>
|
||
Pingback: <a href="https://www.arnoldit.com/wordpress/2025/12/04/titanic-talk-this-ship-will-not-fail/" class="url" rel="ugc external nofollow">Titanic Talk: This Ship Will Not Fail : Stephen E. Arnold @ Beyond Search</a> </div>
|
||
</li><!-- #comment-## -->
|
||
</ol><!-- .commentlist -->
|
||
|
||
|
||
|
||
|
||
<div id="respond" class="comment-respond">
|
||
<h3 id="reply-title" class="comment-reply-title">Leave a Reply <small><a rel="nofollow" id="cancel-comment-reply-link" href="/2025/06/21/big-enough-to-fail/#respond" style="display:none;">Cancel reply</a></small></h3><form action="https://willgallego.com/wp-comments-post.php" method="post" id="commentform" class="comment-form"><p class="comment-notes"><span id="email-notes">Your email address will not be published.</span> <span class="required-field-message">Required fields are marked <span class="required">*</span></span></p><p class="comment-form-comment"><label for="comment">Comment <span class="required">*</span></label> <textarea id="comment" name="comment" cols="45" rows="8" maxlength="65525" required></textarea></p><p class="comment-form-author"><label for="author">Name <span class="required">*</span></label> <input id="author" name="author" type="text" value="" size="30" maxlength="245" autocomplete="name" required /></p>
|
||
<p class="comment-form-email"><label for="email">Email <span class="required">*</span></label> <input id="email" name="email" type="email" value="" size="30" maxlength="100" aria-describedby="email-notes" autocomplete="email" required /></p>
|
||
<p class="comment-form-url"><label for="url">Website</label> <input id="url" name="url" type="url" value="" size="30" maxlength="200" autocomplete="url" /></p>
|
||
<p class="form-submit"><input name="submit" type="submit" id="submit" class="submit" value="Post Comment" /> <input type='hidden' name='comment_post_ID' value='800' id='comment_post_ID' />
|
||
<input type='hidden' name='comment_parent' id='comment_parent' value='0' />
|
||
</p><p style="display: none;"><input type="hidden" id="akismet_comment_nonce" name="akismet_comment_nonce" value="a6595e7871" /></p><p style="display: none !important;" class="akismet-fields-container" data-prefix="ak_"><label>Δ<textarea name="ak_hp_textarea" cols="45" rows="8" maxlength="100"></textarea></label><input type="hidden" id="ak_js_1" name="ak_js" value="153"/><script>document.getElementById( "ak_js_1" ).setAttribute( "value", ( new Date() ).getTime() );</script></p></form> </div><!-- #respond -->
|
||
</div><!-- #comments .comments-area -->
|
||
</div>
|
||
<div id="secondary" class="col-md-4" role="complementary">
|
||
|
||
<aside id="recent-posts-4" class="widget widget_recent_entries">
|
||
<h3 class="widget-title">Recent Posts</h3>
|
||
<ul>
|
||
<li>
|
||
<a href="https://willgallego.com/2025/06/21/big-enough-to-fail/" aria-current="page">Big Enough to Fail</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/">An Incident Review of an Incident Review</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://willgallego.com/2024/11/11/job-switching-and-titles/">Job Switching and Titles</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://willgallego.com/2024/07/12/the-fridge-theory-of-blameless-organizations/">The Fridge Theory of Blameless Organizations</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</aside><aside id="search-2" class="widget widget_search"><form role="search" method="get" class="search-form" action="https://willgallego.com/">
|
||
<label>
|
||
<span class="screen-reader-text">Search for:</span>
|
||
<input type="search" class="search-field" placeholder="Search …" value="" name="s" />
|
||
</label>
|
||
<input type="submit" class="search-submit" value="Search" />
|
||
</form></aside>
|
||
<aside id="recent-posts-2" class="widget widget_recent_entries">
|
||
<h3 class="widget-title">Recent Posts</h3>
|
||
<ul>
|
||
<li>
|
||
<a href="https://willgallego.com/2025/06/21/big-enough-to-fail/" aria-current="page">Big Enough to Fail</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/">An Incident Review of an Incident Review</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://willgallego.com/2024/11/11/job-switching-and-titles/">Job Switching and Titles</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://willgallego.com/2024/07/12/the-fridge-theory-of-blameless-organizations/">The Fridge Theory of Blameless Organizations</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://willgallego.com/2024/06/21/how-many-maybes-until-empathy/">How Many Maybe’s until Empathy?</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</aside><aside id="recent-comments-2" class="widget widget_recent_comments"><h3 class="widget-title">Recent Comments</h3><ul id="recentcomments"><li class="recentcomments"><span class="comment-author-link"><a href="https://www.arnoldit.com/wordpress/2025/12/04/titanic-talk-this-ship-will-not-fail/" class="url" rel="ugc external nofollow">Titanic Talk: This Ship Will Not Fail : Stephen E. Arnold @ Beyond Search</a></span> on <a href="https://willgallego.com/2025/06/21/big-enough-to-fail/#comment-37339">Big Enough to Fail</a></li><li class="recentcomments"><span class="comment-author-link"><a href="https://sreweekly.com/sre-weekly-issue-484/" class="url" rel="ugc external nofollow">SRE Weekly Issue #484 – SRE WEEKLY</a></span> on <a href="https://willgallego.com/2025/06/21/big-enough-to-fail/#comment-25648">Big Enough to Fail</a></li><li class="recentcomments"><span class="comment-author-link">Jacob Rose</span> on <a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/#comment-17389">An Incident Review of an Incident Review</a></li><li class="recentcomments"><span class="comment-author-link">Leonardo Camara</span> on <a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/#comment-16366">An Incident Review of an Incident Review</a></li><li class="recentcomments"><span class="comment-author-link"><a href="https://sreweekly.com/sre-weekly-issue-460/" class="url" rel="ugc external nofollow">SRE Weekly Issue #460 – SRE WEEKLY</a></span> on <a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/#comment-16284">An Incident Review of an Incident Review</a></li></ul></aside><aside id="archives-2" class="widget widget_archive"><h3 class="widget-title">Archives</h3>
|
||
<ul>
|
||
<li><a href='https://willgallego.com/2025/06/'>June 2025</a></li>
|
||
<li><a href='https://willgallego.com/2025/01/'>January 2025</a></li>
|
||
<li><a href='https://willgallego.com/2024/11/'>November 2024</a></li>
|
||
<li><a href='https://willgallego.com/2024/07/'>July 2024</a></li>
|
||
<li><a href='https://willgallego.com/2024/06/'>June 2024</a></li>
|
||
<li><a href='https://willgallego.com/2024/03/'>March 2024</a></li>
|
||
<li><a href='https://willgallego.com/2024/02/'>February 2024</a></li>
|
||
<li><a href='https://willgallego.com/2024/01/'>January 2024</a></li>
|
||
<li><a href='https://willgallego.com/2023/10/'>October 2023</a></li>
|
||
<li><a href='https://willgallego.com/2023/04/'>April 2023</a></li>
|
||
<li><a href='https://willgallego.com/2023/01/'>January 2023</a></li>
|
||
<li><a href='https://willgallego.com/2022/12/'>December 2022</a></li>
|
||
<li><a href='https://willgallego.com/2022/06/'>June 2022</a></li>
|
||
<li><a href='https://willgallego.com/2022/02/'>February 2022</a></li>
|
||
<li><a href='https://willgallego.com/2022/01/'>January 2022</a></li>
|
||
<li><a href='https://willgallego.com/2021/08/'>August 2021</a></li>
|
||
<li><a href='https://willgallego.com/2021/05/'>May 2021</a></li>
|
||
<li><a href='https://willgallego.com/2021/02/'>February 2021</a></li>
|
||
<li><a href='https://willgallego.com/2021/01/'>January 2021</a></li>
|
||
<li><a href='https://willgallego.com/2020/09/'>September 2020</a></li>
|
||
<li><a href='https://willgallego.com/2020/06/'>June 2020</a></li>
|
||
<li><a href='https://willgallego.com/2020/05/'>May 2020</a></li>
|
||
<li><a href='https://willgallego.com/2020/02/'>February 2020</a></li>
|
||
<li><a href='https://willgallego.com/2019/12/'>December 2019</a></li>
|
||
<li><a href='https://willgallego.com/2019/08/'>August 2019</a></li>
|
||
<li><a href='https://willgallego.com/2019/07/'>July 2019</a></li>
|
||
<li><a href='https://willgallego.com/2019/04/'>April 2019</a></li>
|
||
<li><a href='https://willgallego.com/2019/02/'>February 2019</a></li>
|
||
<li><a href='https://willgallego.com/2018/12/'>December 2018</a></li>
|
||
<li><a href='https://willgallego.com/2018/08/'>August 2018</a></li>
|
||
<li><a href='https://willgallego.com/2018/07/'>July 2018</a></li>
|
||
<li><a href='https://willgallego.com/2018/04/'>April 2018</a></li>
|
||
<li><a href='https://willgallego.com/2018/02/'>February 2018</a></li>
|
||
<li><a href='https://willgallego.com/2018/01/'>January 2018</a></li>
|
||
<li><a href='https://willgallego.com/2017/11/'>November 2017</a></li>
|
||
<li><a href='https://willgallego.com/2017/10/'>October 2017</a></li>
|
||
<li><a href='https://willgallego.com/2017/09/'>September 2017</a></li>
|
||
</ul>
|
||
|
||
</aside> </div><!-- #secondary.widget-area --> </div>
|
||
</div>
|
||
|
||
</main><!-- main -->
|
||
|
||
<footer id="footer" role="contentinfo">
|
||
<div id="footer-content" class="container">
|
||
<div class="row">
|
||
<div class="copyright col-lg-12">
|
||
<span class="pull-left">Copyright © 2026 <a href="https://willgallego.com">Will Gallego</a>. All Rights Reserved.</span>
|
||
<span class="credit-link pull-right"><i class="fa fa-leaf"></i>The Arcade Basic Theme by <a href="https://themes.bavotasan.com/themes/arcade">bavotasan.com</a>.</span>
|
||
</div><!-- .col-lg-12 -->
|
||
</div><!-- .row -->
|
||
</div><!-- #footer-content.container -->
|
||
</footer><!-- #footer -->
|
||
</div><!-- #page -->
|
||
|
||
<script type="speculationrules">
|
||
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"\/*"},{"not":{"href_matches":["\/wp-*.php","\/wp-admin\/*","\/wp-content\/uploads\/*","\/wp-content\/*","\/wp-content\/plugins\/*","\/wp-content\/themes\/arcade-basic\/*","\/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
|
||
</script>
|
||
<script type="text/javascript" src="https://willgallego.com/wp-includes/js/comment-reply.min.js?ver=6.8.8" id="comment-reply-js" async="async" data-wp-strategy="async"></script>
|
||
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/bootstrap.min.js?ver=3.0.3" id="bootstrap-js"></script>
|
||
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/fillsize.js?ver=6.8.8" id="fillsize-js"></script>
|
||
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/jquery.arctext.js?ver=6.8.8" id="arctext-js"></script>
|
||
<script type="text/javascript" id="arcade-basic-script-js-extra">
|
||
/* <![CDATA[ */
|
||
var arcade_basic_vars = {"arc":"800","fittext":""};
|
||
/* ]]> */
|
||
</script>
|
||
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/theme.js?ver=6.8.8" id="arcade-basic-script-js"></script>
|
||
<script defer type="text/javascript" src="https://willgallego.com/wp-content/plugins/akismet/_inc/akismet-frontend.js?ver=1758810644" id="akismet-frontend-js"></script>
|
||
</body>
|
||
</html>
|
||
|
||
<!-- Page cached by LiteSpeed Cache 7.5.0.1 on 2026-09-11 02:09:50 --> |