Files
nexus/sreweekly/articles/34/02-high-tempo-high-consequence.html
2026-09-12 17:23:01 +08:00

254 lines
40 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>High Tempo, High Consequence &#8211; Kitchen Soap</title>
<meta name='robots' content='max-image-preview:large' />
<link rel="alternate" type="application/rss+xml" title="Kitchen Soap &raquo; Feed" href="https://www.kitchensoap.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="Kitchen Soap &raquo; Comments Feed" href="https://www.kitchensoap.com/comments/feed/" />
<link rel="alternate" type="application/rss+xml" title="Kitchen Soap &raquo; High Tempo, High Consequence Comments Feed" href="https://www.kitchensoap.com/2014/03/13/high-tempo-high-consequence/feed/" />
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://www.kitchensoap.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fwww.kitchensoap.com%2F2014%2F03%2F13%2Fhigh-tempo-high-consequence%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://www.kitchensoap.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fwww.kitchensoap.com%2F2014%2F03%2F13%2Fhigh-tempo-high-consequence%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style*=border-color]){border-style:solid}html :where([style*=border-top-color]){border-top-style:solid}html :where([style*=border-right-color]){border-right-style:solid}html :where([style*=border-bottom-color]){border-bottom-style:solid}html :where([style*=border-left-color]){border-left-style:solid}html :where([style*=border-width]){border-style:solid}html :where([style*=border-top-width]){border-top-style:solid}html :where([style*=border-right-width]){border-right-style:solid}html :where([style*=border-bottom-width]){border-bottom-style:solid}html :where([style*=border-left-width]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--font-family--libre-baskerville: "Libre Baskerville", serif;--wp--preset--font-family--jost: Jost, sans-serif;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}.has-libre-baskerville-font-family{font-family: var(--wp--preset--font-family--libre-baskerville) !important;}.has-jost-font-family{font-family: var(--wp--preset--font-family--jost) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='loupely-canvas-lite-css' href='https://www.kitchensoap.com/wp-content/cache/autoptimize/css/autoptimize_single_81547c2bf1b39fd0b27f64bee28baf0a.css?ver=1.2.3' media='all' />
<link rel='stylesheet' id='lclite-design-css' href='https://www.kitchensoap.com/wp-content/cache/autoptimize/css/autoptimize_single_768cee5b62c05603aae6f831a2301733.css?ver=1.2.3' media='all' />
<link rel="https://api.w.org/" href="https://www.kitchensoap.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://www.kitchensoap.com/wp-json/wp/v2/posts/9474" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://www.kitchensoap.com/xmlrpc.php?rsd" />
<link rel="canonical" href="https://www.kitchensoap.com/2014/03/13/high-tempo-high-consequence/" />
<link rel='shortlink' href='https://www.kitchensoap.com/?p=9474' />
<style class="wp-fonts-local">
@font-face{font-family:"Libre Baskerville";font-style:normal;font-weight:400;font-display:fallback;src:url('http://www.kitchensoap.com/wp-content/uploads/fonts/kmKUZrc3Hgbbcjq75U4uslyuy4kn0olVQ-LglH6T17uj8Q4SDAlOvuPCfQ.woff2') format('woff2');}
@font-face{font-family:"Libre Baskerville";font-style:normal;font-weight:700;font-display:fallback;src:url('http://www.kitchensoap.com/wp-content/uploads/fonts/kmKUZrc3Hgbbcjq75U4uslyuy4kn0olVQ-LglH6T17ujFgkSDAlOvuPCfQ.woff2') format('woff2');}
@font-face{font-family:"Libre Baskerville";font-style:italic;font-weight:400;font-display:fallback;src:url('http://www.kitchensoap.com/wp-content/uploads/fonts/kmKWZrc3Hgbbcjq75U4uslyuy4kn0qNccR04_RUJeby2OU36SgNPnObSfbge.woff2') format('woff2');}
@font-face{font-family:"Libre Baskerville";font-style:italic;font-weight:700;font-display:fallback;src:url('http://www.kitchensoap.com/wp-content/uploads/fonts/kmKWZrc3Hgbbcjq75U4uslyuy4kn0qNccR04_RUJeby2OU0dTQNPnObSfbge.woff2') format('woff2');}
@font-face{font-family:Jost;font-style:normal;font-weight:400;font-display:fallback;src:url('http://www.kitchensoap.com/wp-content/uploads/fonts/92zPtBhPNqw79Ij1E865zBUv7myjJQVDPokMmuHL.woff2') format('woff2');}
@font-face{font-family:Jost;font-style:normal;font-weight:700;font-display:fallback;src:url('http://www.kitchensoap.com/wp-content/uploads/fonts/92zPtBhPNqw79Ij1E865zBUv7mxEIgVDPokMmuHL.woff2') format('woff2');}
@font-face{font-family:Jost;font-style:italic;font-weight:400;font-display:fallback;src:url('http://www.kitchensoap.com/wp-content/uploads/fonts/92zJtBhPNqw73oHH7BbQp4-B6XlrZu0FNIgun_HLMEo.woff2') format('woff2');}
@font-face{font-family:Jost;font-style:italic;font-weight:700;font-display:fallback;src:url('http://www.kitchensoap.com/wp-content/uploads/fonts/92zJtBhPNqw73oHH7BbQp4-B6XlrZgoCNIgun_HLMEo.woff2') format('woff2');}
</style>
<!-- BEGIN Adobe Fonts for WordPress --><!-- END Adobe Fonts for WordPress --></head>
<body class="wp-singular post-template-default single single-post postid-9474 single-format-standard wp-embed-responsive wp-theme-loupely-canvas-lite">
<a class="lclite-skip-link screen-reader-text" href="#lclite-main-content">Skip to content</a>
<header class="lclite-site-header">
<div class="lclite-header-inner">
<div class="lclite-branding">
<a class="lclite-site-title" href="https://www.kitchensoap.com/" rel="home">Kitchen Soap</a><p class="lclite-tagline">Thoughts on systems safety, software operations, and sociotechnical systems.</p> </div>
<nav class="lclite-nav" aria-label="Primary">
<ul id="menu-menu-1" class="menu"><li id="menu-item-9440" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-9440"><a href="https://www.kitchensoap.com/about-me/">About Me</a></li>
<li id="menu-item-9441" class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-9441"><a href="https://www.kitchensoap.com/category/talks/">Talks</a></li>
<li id="menu-item-9444" class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-9444"><a href="https://www.kitchensoap.com/category/books/">Books</a></li>
<li id="menu-item-9501" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-9501"><a href="https://www.kitchensoap.com/category/complex-systems/">Complex Systems</a></li>
<li id="menu-item-9502" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-9502"><a href="https://www.kitchensoap.com/category/human-factors/">Human Factors</a></li>
<li id="menu-item-9503" class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-9503"><a href="https://www.kitchensoap.com/category/webops/">WebOps</a></li>
</ul> </nav>
</div>
</header>
<main id="lclite-main-content" class="lclite-content">
<div class="lclite-blog">
<article id="post-9474" class="lclite-post post-9474 post type-post status-publish format-standard hentry category-complex-systems category-human-factors category-systems-safety tag-complex-systems-2 tag-human-factors-2 tag-systems-safety-2">
<header class="lclite-entry-header">
<h1 class="lclite-entry-title">High Tempo, High Consequence</h1> <p class="lclite-entry-meta">
<span class="lclite-posted-on">March 13, 2014</span>
<span class="lclite-byline">
by allspaw </span>
</p>
</header>
<div class="lclite-entry-content lclite-prose">
<h3>A Time to Remember</h3>
<p>I want you to think back to a time when you found yourself in an emergency situation at work.</p>
<p>Maybe it was diagnosing and trying to recover from a site outage.<br />
Maybe it was when you were confronting the uncertain possibility of critical data loss.<br />
Maybe it was when you and your team were responding to a targeted and malicious attack.</p>
<p>Maybe it was a time when you, maybe even milliseconds after you triggered some action (maybe even just hit “enter” after a command), realized that you just made a terrible mistake and inadvertently kicked off irreparable destruction that cannot be undone.</p>
<p>Maybe it was a shocking discovery that something bad (silent data corruption, for example) has been happening for a long time and no one knew it was happening.</p>
<p>Maybe it was a time when silence descended upon your team as they tried to understand what was happening to the site, and the business. The time when you didn’t even know what was going on, forget about hypothesizing how to go about fixing it.</p>
<p>Think back to the time when you had to actively hold back the fears of what the news headlines or management or your board of directors were going to say about it when it was over, because you have a job to do and worrying about those things wouldn’t bring the site back up.</p>
<p>Think back to a time when after you’ve resolved an outage and the dust has settled, your adrenaline turns its focus to amplifying the fear that you and your team will have<em> no idea when that will happen again</em> in the future because you’re still uncertain how it happened in the first place.</p>
<p>I’ve been working in web operations for over 15 years, and I can describe in excruciating detail examples of many of those situations. Many of my peers can tell stories like those, and often do. I’m willing to bet that you too, dear reader, will find these to be familiar feelings.</p>
<p>Those moments are <em><strong>real</strong></em>.<br />
The cortisol coursing through your body during those times was <em><strong>real</strong></em>.<br />
The effect of time pressure is <em><strong>real</strong></em>.</p>
<p>The problems that show up when you have to make critical decisions based on incredibly incomplete information is real.</p>
<p>The issues that show up when having to communicate effectively across multiple team members, sometimes separated by time (as people enter and exit the response to an outage) as well as distance (connected through chat or audio/video conferencing) are all real.</p>
<p>The issues when coordinating who is going to do what, and when they’re going to do it, and confirming that whatever they did went well enough for someone else to do their part next, etc. are all real.</p>
<p>And they all are real <strong><i>regardless of the outcomes of the scenarios.</i></strong><i></i></p>
<p>&nbsp;</p>
<h3>Comparisons</h3>
<p>Those moments do happen in other domains. Other domains like healthcare, where nurses work in neonatal intensive care units.</p>
<p>Like infantry in battle.<br />
Like ground control in a mission control organization.<br />
Like a regional railway control center.<br />
Like a trauma surgeon in an operating room.<br />
Like an air traffic controller.<br />
Like a pilot, just flying.<br />
Like a wildland firefighting hotshow crew.<br />
Like a ship crew.</p>
<p>Like a software engineer working in a high-frequency trading company.</p>
<p>All of those domains (and many others) have these in common:</p>
<ul>
<li>They need to make decisions and take action under time pressure and with <em>incomplete</em> information, and when the results have just as much potential to make things <i>worse</i> than they do to make things better.</li>
<li>They have to communicate a lot of information and coordinate actions between teams and team members in the shortest time possible, while also not missing critical details.</li>
<li>They all work in areas where small changes can bring out large results whose potential for surprising everyone is quite high.</li>
<li>They all work in organizations whose cultural, social, hierarchical, and decision-making norms are influenced by past successes and failures, many of which manifest in these high-tempo scenarios.</li>
</ul>
<p><span style="line-height: 1.5em;">But: do the people in those domains experience those moments differently?</span></p>
<p>In other words: does a nurse or air traffic controller’s experience in those real moments differ from ours, because lives are at stake?</p>
<p>Do they experience <i>more</i> stress? Different stress?<br />
Do they navigate <a title="Alarms in Anesthesia" href="http://fhs.mcmaster.ca/anesthesiaresearch/documents/ERGOArticle.pdf" target="_blank" rel="noopener">alerts, alarms, and computers</a> in more prudent or better ways than we do?<br />
Do they have <i>more </i>problems with communications and coordinating response amongst multiple team members?<br />
Are they measurably <strong><em>more careful </em></strong>in their work because of the stakes?</p>
<p>Are all of their decisions perfectly clear, or do they have to navigate ambiguity sometimes, just like we do?<br />
Because there are lives to protect, is their decision-making in high-tempo scenarios different? Better?</p>
<p>My assertion is that high-tempo/high-consequence scenarios in the domain of Internet engineering and operations do indeed have similarities with those other domains, and that understanding all of those dynamics, pitfalls, learning opportunities, etc. is critical for the future.</p>
<p>All of the future.</p>
<p>Do these scenarios yield the same results, organizationally, in those domains as they do in web engineering and operations? Likely not. But I will add that unless we attempt to understand those similarities and differences, we’re not going to know what to learn from, and what to discard.</p>
<p>Hrm. Really?</p>
<p>Because how can we compare something like the Site Reliability Engineer team’s experience at Google.com to something like the air traffic control crew experience landing airplanes at Heathrow?</p>
<p>I have two responses to this question.</p>
<p>The first is that we’re assuming that the <em>potential </em>severity of the consequence influences the way people (and teams of people) think, act, and behave under those conditions. Research on how people behave <a title="Escalating Scenarios" href="http://lup.lub.lu.se/luur/download?func=downloadFile&amp;recordOId=2540835&amp;fileOId=2540838" target="_blank" rel="noopener">under uncertain conditions and escalating scenarios</a> do indeed have generalizable findings across many domains.</p>
<p>The second is that in trivializing the comparison to loss of life versus non-loss of life, we can underestimate the n-order effects that the Internet can have on geopolitical, economic, and other areas that are further away from servers and network cables. We would be too reductionist in our thinking. The Internet is not just about photos of cats. It bolsters <a title="Twitter Iran" href="http://www.reuters.com/article/2009/06/16/us-iran-election-twitter-usa-idUSWBT01137420090616" target="_blank" rel="noopener">elections in emerging democracies</a>, <a title="Arab Spring" href="http://www.wired.com/magazine/2013/04/arabspring/" target="_blank" rel="noopener">revolutions</a>, and a whole host of other things that <a href="http://www.wired.com/gadgetlab/2014/03/twitter-big-fail-whale-now/" target="_blank" rel="noopener">prove to be life-critical</a>.</p>
<h3>A View From Not Too Far Away</h3>
<p>At the Velocity Conference in 2012, Dr. Richard Cook (an anesthesiologist and one of the most forward-thinking men I know in these areas), was <a title="Mac Slocum interviews Richard Cook" href="http://www.youtube.com/watch?v=R_PDc0HFdP0" target="_blank" rel="noopener">interviewed after his keynote by Mac Slocum, from O’Reilly</a>.</p>
<p>Mac, hoping to contrast Cook’s normal audience to that of Velocity’s, asked about whether or not he saw crossover from the “safety-critical” domains to that of web operations:</p>
<p><strong>Cook: “</strong>Anytime you find a world in which you have high consequences, high tempo, time pressure, and lots of complexity, <i>semantic </i>complexity<i>, underlying deep </i>complexity, and people are called upon to manage that you’re going to have these kinds of issues arise. And the general model that we have is one for <i>systems</i>, not for specific instances of systems. So I kind of expected that it would work&#8230;”</p>
<p><strong>Mac: </strong>”&#8230;obviously failure in the health care world is different than failure in the [web operations] world. What is the right way to address failure, the appropriate way to address failure? Because obviously you shouldn’t have people in this space who are assigning the same level of importance to failure as you would?”</p>
<p><strong>Cook: “</strong>You really think so?”</p>
<p><strong>Mac</strong>: “Well, if a computer goes down, that’s one thing.”</p>
<p><strong>Cook</strong>: “If you lose $300 to $400 million dollars, don’t you think that would buy a lot of vaccines?”</p>
<p><strong>Mac</strong>: “[laughs] well, that’s true.”</p>
<p><strong>Cook</strong>: “Look, the fact that it appears to be dramatic because we’re in the operating room or the intensive care unit doesn’t change the importance of what people are doing. That’s a consequence of being close to and seeing things in a dramatic fashion. But what’s happening here? This is the <i>lifeblood</i> of commerce. This is the core of the economic engine that we’re now experiencing. You think that’s not important?”</p>
<p><strong>Mac</strong>: “So it’s ok then, to assign deep importance to this work?”</p>
<p><strong>Cook</strong>: “Yeah, I think the big question will be whether or not we are actually able to conclude the healthcare importance measures up to the importance web ops, not the other way around.”<i><br />
</i></p>
<p>Richard further mentioned in his keynote last year at New York’s Velocity that:</p>
<blockquote><p>“&#8230;web applications have a tendency to become business critical applications, and business-critical applications have a tendency to become safety-critical systems.”</p></blockquote>
<p>And yes, software bugs <a title="Toyota stack overflow" href="http://embeddedgurus.com/state-space/2014/02/are-we-shooting-ourselves-in-the-foot-with-stack-overflow/" target="_blank" rel="noopener"><i>have </i>killed people.</a></p>
<p>When I began <a title="Always a Student: Operations and Systems Safety" href="http://www.kitchensoap.com/2013/05/28/always-a-student-operations-and-systems-safety/" target="_blank" rel="noopener">my studies at Lund University</a>, I was joined by practitioners in many of those domains: air traffic control, aviation, wildland fire, child welfare services, mining, oil and gas industry, submarine safety, and maritime accident investigation.</p>
<p>I will admit at the first learning lab of my course, I mentioned that I felt like a bit of an outsider (or at least a cheater in getting away with failures that don’t kill people) and one of my classmates responded:</p>
<blockquote><p>“John, why do you think that understanding these scenarios and potentially improving upon them has anything to do with body count? Do you think that our organizations are influenced <i>more</i> by body count than commercial and economic influences? Complex failures don’t care about how many dollars or bodies you will lose — they are equal opportunists.”</p></blockquote>
<p>I now understand this.</p>
<p>So don’t be fooled into thinking that those human moments at the beginning of this post are any different in other domains, or that our responsibility to understand complex system failures is less important in web engineering and operations than it is elsewhere.</p>
</div>
<footer class="lclite-entry-footer">
<p class="lclite-cats"><a href="https://www.kitchensoap.com/category/complex-systems/" rel="category tag">Complex Systems</a>, <a href="https://www.kitchensoap.com/category/human-factors/" rel="category tag">Human Factors</a>, <a href="https://www.kitchensoap.com/category/systems-safety/" rel="category tag">Systems Safety</a></p>
<p class="lclite-tags"><a href="https://www.kitchensoap.com/tag/complex-systems-2/" rel="tag">complex systems</a>, <a href="https://www.kitchensoap.com/tag/human-factors-2/" rel="tag">human factors</a>, <a href="https://www.kitchensoap.com/tag/systems-safety-2/" rel="tag">systems safety</a></p>
</footer>
</article>
<nav class="navigation post-navigation" aria-label="Posts">
<h2 class="screen-reader-text">Post navigation</h2>
<div class="nav-links"><div class="nav-previous"><a href="https://www.kitchensoap.com/2014/02/01/we-are-too-much-accustomed-to-attribute-to-a-singl/" rel="prev"><span class="lclite-nav-label">Previous post</span> We are too much accustomed to attribute to a singl&#8230;</a></div><div class="nav-next"><a href="https://www.kitchensoap.com/2014/03/19/paradigm-check-point-prefacing-debriefings/" rel="next"><span class="lclite-nav-label">Next post</span> Paradigm Check Point: Prefacing Debriefings</a></div></div>
</nav><div id="comments" class="lclite-comments">
<h2 class="lclite-comments-title">
3 comments </h2>
<ol class="lclite-comment-list">
<li id="comment-14122" class="comment even thread-even depth-1">
<article id="div-comment-14122" class="comment-body">
<footer class="comment-meta">
<div class="comment-author vcard">
<b class="fn">Con Kenney</b> <span class="says">says:</span> </div><!-- .comment-author -->
<div class="comment-metadata">
<a href="https://www.kitchensoap.com/2014/03/13/high-tempo-high-consequence/#comment-14122"><time datetime="2014-04-01T08:01:34-08:00">April 1, 2014 at 8:01 am</time></a> </div><!-- .comment-metadata -->
</footer><!-- .comment-meta -->
<div class="comment-content">
<p>I&#8217;ve reached similar conclusions about the similarity between protecting information systems infrastructure and the work of other high-reliability organizations. I&#8217;d be happy to send you a briefing for the CIO of U.S. Department of the Navy or a list of references if you&#8217;d find that useful.</p>
</div><!-- .comment-content -->
</article><!-- .comment-body -->
</li><!-- #comment-## -->
<li id="comment-14126" class="comment byuser comment-author-admin bypostauthor odd alt thread-odd thread-alt depth-1">
<article id="div-comment-14126" class="comment-body">
<footer class="comment-meta">
<div class="comment-author vcard">
<b class="fn"><a href="http://www.kitchensoap.com/" class="url" rel="ugc">allspaw</a></b> <span class="says">says:</span> </div><!-- .comment-author -->
<div class="comment-metadata">
<a href="https://www.kitchensoap.com/2014/03/13/high-tempo-high-consequence/#comment-14126"><time datetime="2014-04-02T03:17:43-08:00">April 2, 2014 at 3:17 am</time></a> </div><!-- .comment-metadata -->
</footer><!-- .comment-meta -->
<div class="comment-content">
<p>Hey Con: Thanks for the comment! Yes, I&#8217;d love to see that! My email address is my last name at Google mail&#8217;s domain name. 🙂</p>
</div><!-- .comment-content -->
</article><!-- .comment-body -->
</li><!-- #comment-## -->
<li id="comment-80905" class="pingback even thread-even depth-1">
<div class="comment-body">
Pingback: <a href="http://sreweekly.com/sre-weekly-issue-34/" class="url" rel="ugc external nofollow">SRE Weekly Issue #34 &#8211; SRE WEEKLY</a> </div>
</li><!-- #comment-## -->
</ol>
<p class="lclite-no-comments">Comments are closed.</p>
</div>
</div>
</main>
<footer class="lclite-site-footer">
<div class="lclite-footer-inner">
<span class="lclite-footer-title">Kitchen Soap</span>
<p class="lclite-colophon">
&copy; 2026 Kitchen Soap </p>
</div>
</footer>
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/loupely-canvas-lite/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://www.kitchensoap.com/wp-includes/js/wp-emoji-release.min.js?ver=7.0.4"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://www.kitchensoap.com/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>