1379 lines
56 KiB
HTML
1379 lines
56 KiB
HTML
<!DOCTYPE html>
|
|
<html dir='ltr' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'>
|
|
<head>
|
|
<link href='https://www.blogger.com/static/v1/widgets/4165133002-widget_css_bundle.css' rel='stylesheet' type='text/css'/>
|
|
<meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/>
|
|
<meta content='blogger' name='generator'/>
|
|
<link href='https://sysadvent.blogspot.com/favicon.ico' rel='icon' type='image/x-icon'/>
|
|
<link href='https://sysadvent.blogspot.com/2017/12/day-3-root-cause-is-plural.html' rel='canonical'/>
|
|
<link rel="alternate" type="application/atom+xml" title="sysadvent - Atom" href="https://sysadvent.blogspot.com/feeds/posts/default" />
|
|
<link rel="alternate" type="application/rss+xml" title="sysadvent - RSS" href="https://sysadvent.blogspot.com/feeds/posts/default?alt=rss" />
|
|
<link rel="service.post" type="application/atom+xml" title="sysadvent - Atom" href="https://draft.blogger.com/feeds/3615332969083650973/posts/default" />
|
|
|
|
<link rel="alternate" type="application/atom+xml" title="sysadvent - Atom" href="https://sysadvent.blogspot.com/feeds/7842984104311049436/comments/default" />
|
|
<!--Can't find substitution for tag [blog.ieCssRetrofitLinks]-->
|
|
<link href='https://upload.wikimedia.org/wikipedia/commons/0/04/Cycads_root.png' rel='image_src'/>
|
|
<meta content='https://sysadvent.blogspot.com/2017/12/day-3-root-cause-is-plural.html' property='og:url'/>
|
|
<meta content='Day 3 - Root Cause is Plural' property='og:title'/>
|
|
<meta content='Written by Cody Wilbourn ( @codywilbourn ) Edited by Kerim Satirli ( @ksatirli ) Post-mortems are an industry standard process that happe...' property='og:description'/>
|
|
<meta content='https://lh3.googleusercontent.com/blogger_img_proxy/AEn0k_vt_1WbGcO-eXmNGsdqNIIxPzwO9CWQOOt4tw4WjWDiCJm9g41mkwmulfkiXCSSk5ZzspYsTuoXZhwCHvm6tFC9TkdhdzAYqXxOI1HrlnSurkMvXuFoUQM04eAIsvVdRpYyRrcOafc=w1200-h630-p-k-no-nu' property='og:image'/>
|
|
<title>
|
|
sysadvent: Day 3 - Root Cause is Plural
|
|
</title>
|
|
<style type='text/css'>@font-face{font-family:'Allerta';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/allerta/v19/TwMO-IAHRlkbx940YnYXSCiN9uc.woff2)format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDRrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcBBrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDBrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+1F00-1FFF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcAxrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0370-0377,U+037A-037F,U+0384-038A,U+038C,U+038E-03A1,U+03A3-03FF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcAhrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0307-0308,U+0590-05FF,U+200C-2010,U+20AA,U+25CC,U+FB1D-FB4F;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDxrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDhrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcABrBdwcoaaQw.woff2)format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;}</style>
|
|
<style id='page-skin-1' type='text/css'><!--
|
|
/*
|
|
-----------------------------------------------
|
|
Blogger Template Style
|
|
Name: Minima
|
|
Designer: Douglas Bowman
|
|
URL: www.stopdesign.com
|
|
Date: 26 Feb 2004
|
|
Updated by: Blogger Team
|
|
----------------------------------------------- */
|
|
/* Variable definitions
|
|
====================
|
|
<Variable name="bgcolor" description="Page Background Color"
|
|
type="color" default="#fff">
|
|
<Variable name="textcolor" description="Text Color"
|
|
type="color" default="#333">
|
|
<Variable name="linkcolor" description="Link Color"
|
|
type="color" default="#58a">
|
|
<Variable name="pagetitlecolor" description="Blog Title Color"
|
|
type="color" default="#666">
|
|
<Variable name="descriptioncolor" description="Blog Description Color"
|
|
type="color" default="#999">
|
|
<Variable name="titlecolor" description="Post Title Color"
|
|
type="color" default="#c60">
|
|
<Variable name="bordercolor" description="Border Color"
|
|
type="color" default="#ccc">
|
|
<Variable name="sidebarcolor" description="Sidebar Title Color"
|
|
type="color" default="#999">
|
|
<Variable name="sidebartextcolor" description="Sidebar Text Color"
|
|
type="color" default="#666">
|
|
<Variable name="visitedlinkcolor" description="Visited Link Color"
|
|
type="color" default="#999">
|
|
<Variable name="bodyfont" description="Text Font"
|
|
type="font" default="normal normal 100% Georgia, Serif">
|
|
<Variable name="headerfont" description="Sidebar Title Font"
|
|
type="font"
|
|
default="normal normal 78% 'Trebuchet MS',Trebuchet,Arial,Verdana,Sans-serif">
|
|
<Variable name="pagetitlefont" description="Blog Title Font"
|
|
type="font"
|
|
default="normal normal 200% Georgia, Serif">
|
|
<Variable name="descriptionfont" description="Blog Description Font"
|
|
type="font"
|
|
default="normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif">
|
|
<Variable name="postfooterfont" description="Post Footer Font"
|
|
type="font"
|
|
default="normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif">
|
|
<Variable name="startSide" description="Side where text starts in blog language"
|
|
type="automatic" default="left">
|
|
<Variable name="endSide" description="Side where text ends in blog language"
|
|
type="automatic" default="right">
|
|
*/
|
|
/* Use this with templates/template-twocol.html */
|
|
body {
|
|
background:#ffffff;
|
|
margin:0;
|
|
color:#333333;
|
|
font:x-small Georgia Serif;
|
|
font-size/* */:/**/small;
|
|
font-size: /**/small;
|
|
text-align: center;
|
|
}
|
|
a:link {
|
|
color:#5588aa;
|
|
text-decoration:none;
|
|
}
|
|
a:visited {
|
|
color:#999999;
|
|
text-decoration:none;
|
|
}
|
|
a:hover {
|
|
color:#cc6600;
|
|
text-decoration:underline;
|
|
}
|
|
a img {
|
|
border-width:0;
|
|
}
|
|
/* Header
|
|
-----------------------------------------------
|
|
*/
|
|
#header-wrapper {
|
|
width:760px;
|
|
margin:0 auto 10px;
|
|
/*border:1px solid #cccccc;*/
|
|
}
|
|
#header-inner {
|
|
background-position: center;
|
|
margin-left: auto;
|
|
margin-right: auto;
|
|
}
|
|
#header {
|
|
margin: 5px;
|
|
/*border: 1px solid #cccccc;*/
|
|
text-align: center;
|
|
color:#666666;
|
|
}
|
|
#header h1 {
|
|
margin:5px 5px 0;
|
|
padding:15px 20px .25em;
|
|
line-height:1.2em;
|
|
text-transform:uppercase;
|
|
letter-spacing:.2em;
|
|
font: normal normal 200% Georgia, Serif;
|
|
}
|
|
#header a {
|
|
color:#666666;
|
|
text-decoration:none;
|
|
}
|
|
#header a:hover {
|
|
color:#666666;
|
|
}
|
|
#header .description {
|
|
margin:0 5px 5px;
|
|
padding:0 20px 15px;
|
|
max-width:800px;
|
|
text-transform:uppercase;
|
|
letter-spacing:.2em;
|
|
line-height: 1.4em;
|
|
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
|
|
color: #999999;
|
|
}
|
|
#header img {
|
|
margin-left: auto;
|
|
margin-right: auto;
|
|
}
|
|
/* Outer-Wrapper
|
|
----------------------------------------------- */
|
|
#outer-wrapper {
|
|
width: 760px;
|
|
margin:0 auto;
|
|
padding:10px;
|
|
text-align:left;
|
|
font: normal normal 100% Arimo;
|
|
}
|
|
#main-wrapper {
|
|
width: 510px;
|
|
float: left;
|
|
word-wrap: break-word; /* fix for long text breaking sidebar float in IE */
|
|
overflow: hidden; /* fix for long non-text content breaking IE sidebar float */
|
|
}
|
|
#sidebar-wrapper {
|
|
width: 220px;
|
|
float: right;
|
|
word-wrap: break-word; /* fix for long text breaking sidebar float in IE */
|
|
overflow: hidden; /* fix for long non-text content breaking IE sidebar float */
|
|
}
|
|
/* Headings
|
|
----------------------------------------------- */
|
|
h2 {
|
|
margin:1.5em 0 .75em;
|
|
font:normal normal 130% Allerta;
|
|
line-height: 1.4em;
|
|
text-transform:uppercase;
|
|
letter-spacing:.2em;
|
|
color:#999999;
|
|
}
|
|
/* Posts
|
|
-----------------------------------------------
|
|
*/
|
|
h2.date-header {
|
|
margin:1.5em 0 .5em;
|
|
}
|
|
.post {
|
|
margin:.5em 0 1.5em;
|
|
border-bottom:1px dotted #cccccc;
|
|
padding-bottom:1.5em;
|
|
}
|
|
.post h3 {
|
|
margin:.25em 0 0;
|
|
padding:0 0 4px;
|
|
font-size:140%;
|
|
font-weight:normal;
|
|
line-height:1.4em;
|
|
color:#cc6600;
|
|
}
|
|
.post h3 a, .post h3 a:visited, .post h3 strong {
|
|
display:block;
|
|
text-decoration:none;
|
|
color:#cc6600;
|
|
font-weight:normal;
|
|
}
|
|
.post h3 strong, .post h3 a:hover {
|
|
color:#333333;
|
|
}
|
|
.post-body {
|
|
margin:0 0 .75em;
|
|
line-height:1.6em;
|
|
}
|
|
.post-body blockquote {
|
|
line-height:1.3em;
|
|
}
|
|
.post-footer {
|
|
margin: .75em 0;
|
|
color:#999999;
|
|
text-transform:uppercase;
|
|
letter-spacing:.1em;
|
|
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
|
|
line-height: 1.4em;
|
|
}
|
|
.comment-link {
|
|
margin-left:.6em;
|
|
}
|
|
.post img {
|
|
padding:4px;
|
|
border:1px solid #cccccc;
|
|
}
|
|
.post blockquote {
|
|
margin:1em 20px;
|
|
}
|
|
.post blockquote p {
|
|
margin:.75em 0;
|
|
}
|
|
.post pre {
|
|
overflow: auto;
|
|
padding-left: 5px;
|
|
padding-right: 5px;
|
|
border: 1px solid #CCCCFF !important;
|
|
}
|
|
.post pre a {
|
|
text-decoration: underline;
|
|
}
|
|
/* Comments
|
|
----------------------------------------------- */
|
|
#comments h4 {
|
|
margin:1em 0;
|
|
font-weight: bold;
|
|
line-height: 1.4em;
|
|
text-transform:uppercase;
|
|
letter-spacing:.2em;
|
|
color: #999999;
|
|
}
|
|
#comments-block {
|
|
margin:1em 0 1.5em;
|
|
line-height:1.6em;
|
|
}
|
|
#comments-block .comment-author {
|
|
margin:.5em 0;
|
|
}
|
|
#comments-block .comment-body {
|
|
margin:.25em 0 0;
|
|
}
|
|
#comments-block .comment-footer {
|
|
margin:-.25em 0 2em;
|
|
line-height: 1.4em;
|
|
text-transform:uppercase;
|
|
letter-spacing:.1em;
|
|
}
|
|
#comments-block .comment-body p {
|
|
margin:0 0 .75em;
|
|
}
|
|
.deleted-comment {
|
|
font-style:italic;
|
|
color:gray;
|
|
}
|
|
#blog-pager-newer-link {
|
|
float: left;
|
|
}
|
|
#blog-pager-older-link {
|
|
float: right;
|
|
}
|
|
#blog-pager {
|
|
text-align: center;
|
|
}
|
|
.feed-links {
|
|
clear: both;
|
|
line-height: 2.5em;
|
|
}
|
|
/* Sidebar Content
|
|
----------------------------------------------- */
|
|
.sidebar {
|
|
color: #666666;
|
|
line-height: 1.5em;
|
|
}
|
|
.sidebar ul {
|
|
list-style:none;
|
|
margin:0 0 0;
|
|
padding:0 0 0;
|
|
}
|
|
.sidebar li {
|
|
margin:0;
|
|
padding-top:0;
|
|
padding-right:0;
|
|
padding-bottom:.25em;
|
|
padding-left:15px;
|
|
text-indent:-15px;
|
|
line-height:1.5em;
|
|
}
|
|
.sidebar .widget, .main .widget {
|
|
border-bottom:1px dotted #cccccc;
|
|
margin:0 0 1.5em;
|
|
padding:0 0 1.5em;
|
|
}
|
|
.main .Blog {
|
|
border-bottom-width: 0;
|
|
}
|
|
/* Profile
|
|
----------------------------------------------- */
|
|
.profile-img {
|
|
float: left;
|
|
margin-top: 0;
|
|
margin-right: 5px;
|
|
margin-bottom: 5px;
|
|
margin-left: 0;
|
|
padding: 4px;
|
|
border: 1px solid #cccccc;
|
|
}
|
|
.profile-data {
|
|
margin:0;
|
|
text-transform:uppercase;
|
|
letter-spacing:.1em;
|
|
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
|
|
color: #999999;
|
|
font-weight: bold;
|
|
line-height: 1.6em;
|
|
}
|
|
.profile-datablock {
|
|
margin:.5em 0 .5em;
|
|
}
|
|
.profile-textblock {
|
|
margin: 0.5em 0;
|
|
line-height: 1.6em;
|
|
}
|
|
.profile-link {
|
|
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
|
|
text-transform: uppercase;
|
|
letter-spacing: .1em;
|
|
}
|
|
/* Footer
|
|
----------------------------------------------- */
|
|
#footer {
|
|
width:760px;
|
|
clear:both;
|
|
margin:0 auto;
|
|
padding-top:15px;
|
|
line-height: 1.6em;
|
|
text-transform:uppercase;
|
|
letter-spacing:.1em;
|
|
text-align: center;
|
|
}
|
|
@media only screen and (max-width:480px) {
|
|
a:link {
|
|
color: #d12a39;
|
|
}
|
|
.mobile-link-button {
|
|
background-color: #d12a39;
|
|
}
|
|
.mobile-link-button a:link, .mobile-link-button a:visited {
|
|
color: #ffffff;
|
|
}
|
|
}
|
|
|
|
--></style>
|
|
<link href='https://draft.blogger.com/dyncss/3615332969083650973/authorization.css?zx=24cd04b2-38ff-45f4-a3f0-f0a07f75ec8f' media='none' onload='if(media!='all')media='all'' rel='stylesheet'/><noscript><link href='https://draft.blogger.com/dyncss/3615332969083650973/authorization.css?zx=24cd04b2-38ff-45f4-a3f0-f0a07f75ec8f' rel='stylesheet'/></noscript>
|
|
<meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/>
|
|
<meta name='google-adsense-platform-domain' content='blogspot.com'/>
|
|
|
|
<!-- data-ad-client=ca-pub-6194074709963145 -->
|
|
|
|
</head>
|
|
<body>
|
|
<div class='navbar section' id='navbar'><div class='widget Navbar' data-version='1' id='Navbar1'><script type="text/javascript">
|
|
function setAttributeOnload(object, attribute, val) {
|
|
if(window.addEventListener) {
|
|
window.addEventListener('load',
|
|
function(){ object[attribute] = val; }, false);
|
|
} else {
|
|
window.attachEvent('onload', function(){ object[attribute] = val; });
|
|
}
|
|
}
|
|
</script>
|
|
<div id="navbar-iframe-container"></div>
|
|
<script type="text/javascript" src="https://apis.google.com/js/platform.js"></script>
|
|
<script type="text/javascript">
|
|
gapi.load("gapi.iframes:gapi.iframes.style.bubble", function() {
|
|
if (gapi.iframes && gapi.iframes.getContext) {
|
|
gapi.iframes.getContext().openChild({
|
|
url: 'https://draft.blogger.com/navbar/3615332969083650973?po\x3d7842984104311049436\x26origin\x3dhttps://sysadvent.blogspot.com',
|
|
where: document.getElementById("navbar-iframe-container"),
|
|
id: "navbar-iframe"
|
|
});
|
|
}
|
|
});
|
|
</script><script type="text/javascript">
|
|
(function() {
|
|
var script = document.createElement('script');
|
|
script.type = 'text/javascript';
|
|
script.src = '//pagead2.googlesyndication.com/pagead/js/google_top_exp.js';
|
|
var head = document.getElementsByTagName('head')[0];
|
|
if (head) {
|
|
head.appendChild(script);
|
|
}})();
|
|
</script>
|
|
</div></div>
|
|
<div id='outer-wrapper'>
|
|
<div id='wrap2'>
|
|
<!-- skip links for text browsers -->
|
|
<span id='skiplinks' style='display:none;'>
|
|
<a href='#main'>
|
|
skip to main
|
|
</a>
|
|
|
|
|
<a href='#sidebar'>
|
|
skip to sidebar
|
|
</a>
|
|
</span>
|
|
<div id='header-wrapper'>
|
|
<div class='header section' id='header'><div class='widget Header' data-version='1' id='Header1'>
|
|
<div id='header-inner'>
|
|
<a href='https://sysadvent.blogspot.com/' style='display: block'>
|
|
<img alt='sysadvent' height='240px; ' id='Header1_headerimg' src='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiYVlIMEU0sXvTBtcY5iFj5QtiT3Klf0S3sgSamkTPsjU1QGvOWHA-6yq173iHys3-ImrIFY26KIPuKEoSNSbitvtnXsZU6b8YPGQ48ir-8Jo2NFrTsaiO_Nl9W9XLt_wyWhTM_16_HeN0/s750/sys-advent-long.png' style='display: block' width='750px; '/>
|
|
</a>
|
|
</div>
|
|
</div></div>
|
|
</div>
|
|
<div id='content-wrapper'>
|
|
<div id='crosscol-wrapper' style='text-align:center'>
|
|
<div class='crosscol no-items section' id='crosscol'></div>
|
|
</div>
|
|
<div id='main-wrapper'>
|
|
<div class='main section' id='main'><div class='widget Blog' data-version='1' id='Blog1'>
|
|
<div class='blog-posts hfeed'>
|
|
<!--Can't find substitution for tag [defaultAdStart]-->
|
|
|
|
<div class="date-outer">
|
|
|
|
<h2 class='date-header'>
|
|
<span>
|
|
December 3, 2017
|
|
</span>
|
|
</h2>
|
|
|
|
<div class="date-posts">
|
|
|
|
<div class='post-outer'>
|
|
<div class='post hentry'>
|
|
<a name='7842984104311049436'></a>
|
|
<h3 class='post-title entry-title'>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-3-root-cause-is-plural.html'>
|
|
Day 3 - Root Cause is Plural
|
|
</a>
|
|
</h3>
|
|
<div class='post-header-line-1'></div>
|
|
<div class='post-body entry-content'>
|
|
<p>Written by Cody Wilbourn (<a href="twitter.com/codywilbourn">@codywilbourn</a>)<br>
|
|
Edited by Kerim Satirli (<a href="twitter.com/ksatirli">@ksatirli</a>)</p>
|
|
|
|
<p>Post-mortems are an industry standard process that happens after incidents and outages as a method of continuous learning and improvement. While the exact format varies from company to company, your post-mortem report typically addresses the Five W’s:</p>
|
|
|
|
<ul>
|
|
<li>What happened?</li>
|
|
<li>Where did it happen?</li>
|
|
<li>Who was impacted by the incident?</li>
|
|
<li>When did problem and resolution events occur?</li>
|
|
<li>Why did the incident occur?</li>
|
|
</ul>
|
|
|
|
<p>The first four questions are generally easy to answer. The question that takes the majority of the time is the <strong>why</strong>. To determine why the incident occurred requires investigative skills, critical thinking, and logical deductions. Sometimes determining the true <em>why</em> takes multiple incidents, as various fixes are attempted before the incident is resolved, but eventually a “root cause” is designated as the root of all the problems and the report is complete.</p>
|
|
|
|
<p>But if your “root cause” amounts to a <em>single</em> failure, you have stopped your process too soon.</p>
|
|
|
|
<h2 id="toc_0">Root Cause is Plural</h2>
|
|
|
|
<p>Like the word <strong>data</strong> is plural, the phrase “root cause” should be as well. There is never a singular <em>root cause</em>, but instead <em>root causes</em> that contribute to an incident. Furthering the plant metaphor, a root came from a seed, and in order to sprout, that seed needed sufficient nutrients, water, and sunlight. Failing to recognize those nutrients, water sources, and sunlight means you prematurely stopped your learning process. There are other roots growing in your field, maybe similar to the one remediated, maybe different, but feeding off of those same needs.</p>
|
|
|
|
<a href="https://upload.wikimedia.org/wikipedia/commons/0/04/Cycads_root.png"><img alt="A network of plant plant roots. CC-SA3.0 retrieved from https://commons.wikimedia.org/wiki/File:Cycads_root.png" src="https://upload.wikimedia.org/wikipedia/commons/0/04/Cycads_root.png" width="95%"></a></p>
|
|
|
|
<p>This isn’t just one root, there’s an entire system of roots.</p>
|
|
|
|
<h2 id="toc_1">Digging into Why</h2>
|
|
|
|
<p>Six Sigma suggests that you ask the question “<a href="https://en.wikipedia.org/wiki/5_Whys">Why?</a>” five times, starting from the problem to determine the root cause. The investigative process you underwent already determined a root cause, but that singular cause is not enough.</p>
|
|
|
|
<p>Dig deeper. Start asking “Why?” from the root cause. This will unearth other reasons, most likely related to decisions and process that started the chain of events</p>
|
|
|
|
<h3 id="toc_2">A Simple Component Failure</h3>
|
|
|
|
<p>Probably one of the easiest apparent failures to diagnose are single component failures, such as a failed hard drive. The part fails, the server goes down, recovery processes follow and the post-mortem states the root cause was “Failed component. Remediated by replacing component.”</p>
|
|
|
|
<p>Do not stop there. Why did the component fail? What was it that contributed to the failure?</p>
|
|
|
|
<p>Did the component degrade over time, because it was in service for too long?</p>
|
|
|
|
<p>Did the component wear out because it was operated outside of its recommended specifications? That is to say, was it stressed by pressure, temperature, orientation, or other physical variables? Was it influenced by other nearby components, such as electromagnetic interference or chemical reactions?</p>
|
|
|
|
<p>Was the component operated within tolerances, but had a manufacturing defect?</p>
|
|
|
|
<p>Did the component have a design defect that rendered it incapable to run in the publicized specifications?</p>
|
|
|
|
<p>For each of those questions you can ask “Why?” again. For example, if the component was operated outside of the allowed tolerances:
|
|
* Why was it operated outside of specifications?
|
|
* Why did the original decision making process select this component for the application?</p>
|
|
|
|
<p>What decisions and processes (or lack thereof) contributed to where things stand today?</p>
|
|
|
|
<p>This example corresponds to the <a href="https://en.wikipedia.org/wiki/Space_Shuttle_Challenger_disaster">Space Shuttle Challenger explosion</a> in 1986.</p>
|
|
|
|
<p>Why did the Challenger explode? A single O-ring failed, causing a chain of events that ultimately led to the explosion.</p>
|
|
|
|
<p><a href="https://upload.wikimedia.org/wikipedia/commons/4/42/STS-51-L_grey_smoke_on_SRB.jpg"><img alt="Gas leak on Space Shuttle Challenger. Public Domain, retrieved from https://en.wikipedia.org/wiki/Space_Shuttle_Challenger_disaster#/media/File:STS-51-L_grey_smoke_on_SRB.jpg" src="https://upload.wikimedia.org/wikipedia/commons/4/42/STS-51-L_grey_smoke_on_SRB.jpg" width="95%"></a></p>
|
|
|
|
<p>Did people throw up their hands and say “this stuff happens when you shoot rockets into space?” No, they didn’t. They, through the Rogers Commission, dug deeper.</p>
|
|
|
|
<p>The O-ring failed due to material deformation caused by low temperatures, lower than what was projected in the engineering phase. That is to say, it was operated outside of specifications.</p>
|
|
|
|
<p>Despite the low temperature, the decision was made to launch. Okay, why? What information was provided for that decision, and how was it generated? Was all of the known information available? Why not? What processes around decision-making allowed that go/no-go decision?</p>
|
|
|
|
<p>Starting with just that initial root cause of an O-ring failure, the Rogers Commission was able to generate a <a href="https://spaceflight.nasa.gov/outreach/SignificantIncidents/assets/rogers_commission_report.pdf">report</a> that resulted in <a href="https://history.nasa.gov/rogersrep/actions.pdf">9 actions for improvement</a>. </p>
|
|
|
|
<p>If you stopped at “the part failed” and state that “things happen”, you missed out all of the learnings that could have happened.</p>
|
|
|
|
<p>Similarly, after the <a href="https://en.wikipedia.org/wiki/Space_Shuttle_Columbia_disaster">Space Shuttle Columbia disaster</a>, caused by a block of foam striking the left wing of the orbiter, the Columbia Accident Investigation Board was able to generate <a href="https://spaceflight.nasa.gov/shuttle/archives/sts-107/investigation/CAIB_medres_full.pdf">a total of 29 recommendations for improvement</a>.</p>
|
|
|
|
<p>What actions would your post-mortem process have resulted in? Ensure the foam doesn’t fall off? Improve the Orbiter so it can handle debris strikes without failure? Surely not all of the 29 recommendations would have been found.</p>
|
|
|
|
<h3 id="toc_3">When To Stop Asking Why</h3>
|
|
|
|
<p>The shuttle disasters were followed by reports that took months to generate after extensive investigation, due at least in part to the loss of life experienced with the disasters. So how much investigation is enough?</p>
|
|
|
|
<p>You stop asking why things happened when you reach a point where things are either completely outside your organization’s control or when it becomes cost prohibitive to address the risk. At some level, things become “acceptable risk” to the business. Sure you can manage a second CDN, or a second or third ISP to get to those last few nines of uptime, but then you realize if Amazon Web Service’s <code>us-east-1</code> region is down, half the Internet is down anyway.</p>
|
|
|
|
<p>But don’t let acceptable risk become a cop-out answer. If this outage was unplanned and people are panicked, then it was not an acceptable risk. One of the root causes would then lie in the process of determining what is acceptable. If the risk was truly acceptable you would be be writing an incident report and not doing a post-mortem analysis attempting to learn from the problems.</p>
|
|
|
|
<p>What is an acceptable risk event like? Several years ago I had this particular 5U server. This server appeared to have been dropped off a truck or conveyor belt by the delivery company, and arrived damaged with the frame bent. While the delivery insurance covered the cost of a replacement server (as the vendor refused to provide warranty coverage on the machine due to the extent of the housing damage and we could no longer use the machine for its original purpose), the machine still booted. So the IT team hammered the frame back in workable shape and the machine was put into service for another less critical use-case. It ran fine for two or three years, until one day it suddenly died. There was no panic, this was an inevitability always planned for. We said to ourselves “Well, today is the day that happened.” There was no post-mortem for the services on the machine. We migrated what had been on that machine and moved on.</p>
|
|
|
|
<p>That’s how you should react to an acceptable risk problem. “Today is the day that finally happened.” That’s not to say there’s not work to be done in a timely fashion after the problem occurred, but there’s no learnings or process improvements to be had. This event was expected from the very beginning.</p>
|
|
|
|
<h2 id="toc_4">Mapping Causes and Effects</h2>
|
|
|
|
<p>Complex investigations require investigating multiple avenues. Mapping the effects and causes in a tree visualization helps understanding of the problems cascade into each other, and may overlap. Visualizations can help promote collaboration – mapping the causes and effects on a whiteboard or with Post-It notes is an easy activity for the post-mortem team to engage in. It’s easy to draw another line and add in more contributing factors as they’re identified. Mapping also ensures that all of the root causes are fully explored and nothing was missed accidentally.</p>
|
|
|
|
<p><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEivpMyPUQ-v2cJEUNgoFPVnIaynlOziDn-07g6XfgojLv-wA8RW5PApntTuP7yK6g6nn1Hd4ykMn-FUSNF_WSiR9ZW32j03qFrdKFl8Qkq5Yj6fuPVX4nvl7rIsVWwnvbSTfskN3OYh1dU/s1600/cause-effect-tree.png"><img src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEivpMyPUQ-v2cJEUNgoFPVnIaynlOziDn-07g6XfgojLv-wA8RW5PApntTuP7yK6g6nn1Hd4ykMn-FUSNF_WSiR9ZW32j03qFrdKFl8Qkq5Yj6fuPVX4nvl7rIsVWwnvbSTfskN3OYh1dU/s320/cause-effect-tree.png" width="95%"/></a></p>
|
|
|
|
<p>The action items or other tasks required following up an incident will be to eliminate as many branches as possible, preferably the leaf nodes or one level up. Eliminating the root of the tree is the same as your current process – it doesn’t ensure that these other contributing root causes won’t impact operations somewhere else in the future. Addressing as much of the problem surface as possible will maximize learnings and improvements, while minimizing the recurrence of issues.</p>
|
|
|
|
<p>As Margaret Hamilton, <a href="https://en.wikipedia.org/wiki/Margaret_Hamilton_(scientist)">Apollo scientist and United States Presidential Medal of Freedom recipient</a> once said, “Learn from everyone and everything, including from that which you would least expect”. <sup><a href="https://www.safaribooksonline.com/library/view/site-reliability-engineering/9781491929117/preface01.html">source</a></sup>. Explore every avenue available to maximize the benefits of the post-mortem process.</p>
|
|
|
|
<div style='clear: both;'></div>
|
|
</div>
|
|
<div class='post-footer'>
|
|
<div class='post-footer-line post-footer-line-1'>
|
|
<span class='post-author vcard'>
|
|
Posted by
|
|
<span class='fn'>
|
|
cwebber
|
|
</span>
|
|
</span>
|
|
<span class='post-timestamp'>
|
|
</span>
|
|
<span class='post-comment-link'>
|
|
</span>
|
|
<span class='post-icons'>
|
|
<span class='item-action'>
|
|
<a href='https://draft.blogger.com/email-post/3615332969083650973/7842984104311049436' title='Email Post'>
|
|
<img alt="" class="icon-action" height="13" src="//www.blogger.com/img/icon18_email.gif" width="18">
|
|
</a>
|
|
</span>
|
|
<span class='item-control blog-admin pid-1452848009'>
|
|
<a href='https://draft.blogger.com/post-edit.g?blogID=3615332969083650973&postID=7842984104311049436&from=pencil' title='Edit Post'>
|
|
<img alt="" class="icon-action" height="18" src="//img2.blogblog.com/img/icon18_edit_allbkg.gif" width="18">
|
|
</a>
|
|
</span>
|
|
</span>
|
|
</div>
|
|
<div class='post-footer-line post-footer-line-2'>
|
|
<span class='post-labels'>
|
|
Labels:
|
|
<a href='https://sysadvent.blogspot.com/search/label/root%20cause' rel='tag'>
|
|
root cause
|
|
</a>
|
|
</span>
|
|
</div>
|
|
<div class='post-footer-line post-footer-line-3'></div>
|
|
</div>
|
|
</div>
|
|
<div class='comments' id='comments'>
|
|
<a name='comments'></a>
|
|
<h4>
|
|
No comments
|
|
:
|
|
</h4>
|
|
<div id='Blog1_comments-block-wrapper'>
|
|
<dl class='' id='comments-block'>
|
|
</dl>
|
|
</div>
|
|
<p class='comment-footer'>
|
|
<a href='https://draft.blogger.com/comment/fullpage/post/3615332969083650973/7842984104311049436' onclick=''>
|
|
Post a Comment
|
|
</a>
|
|
</p>
|
|
<div id='backlinks-container'>
|
|
<div id='Blog1_backlinks-container'>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
</div></div>
|
|
|
|
<!--Can't find substitution for tag [adEnd]-->
|
|
</div>
|
|
<div class='blog-pager' id='blog-pager'>
|
|
<span id='blog-pager-newer-link'>
|
|
<a class='blog-pager-newer-link' href='https://sysadvent.blogspot.com/2017/12/day-4-this-christmas-build-your-own-pki.html' id='Blog1_blog-pager-newer-link' title='Newer Post'>
|
|
Newer Post
|
|
</a>
|
|
</span>
|
|
<span id='blog-pager-older-link'>
|
|
<a class='blog-pager-older-link' href='https://sysadvent.blogspot.com/2017/12/day-2-shifting-left-securely-with-inspec.html' id='Blog1_blog-pager-older-link' title='Older Post'>
|
|
Older Post
|
|
</a>
|
|
</span>
|
|
<a class='home-link' href='https://sysadvent.blogspot.com/'>
|
|
Home
|
|
</a>
|
|
</div>
|
|
<div class='clear'></div>
|
|
<div class='post-feeds'>
|
|
<div class='feed-links'>
|
|
Subscribe to:
|
|
<a class='feed-link' href='https://sysadvent.blogspot.com/feeds/7842984104311049436/comments/default' target='_blank' type='application/atom+xml'>
|
|
Post Comments
|
|
(
|
|
Atom
|
|
)
|
|
</a>
|
|
</div>
|
|
</div>
|
|
</div></div>
|
|
</div>
|
|
<div id='sidebar-wrapper'>
|
|
<div class='sidebar section' id='sidebar'><div class='widget HTML' data-version='1' id='HTML2'>
|
|
<h2 class='title'>
|
|
What is sysadvent?
|
|
</h2>
|
|
<div class='widget-content'>
|
|
<p>
|
|
One article for each day of December, ending on the 25th article.
|
|
|
|
</p><p>
|
|
|
|
With the goals of sharing, openness, and mentoring, we aim to provide great articles about systems administration topics written by fellow sysadmins.
|
|
|
|
</p><p>
|
|
|
|
Want to get involved? Join the <a href="http://groups.google.com/group/sysadvent">mailing list</a>!
|
|
<p></p></p>
|
|
</div>
|
|
<div class='clear'></div>
|
|
</div><div class='widget Subscribe' data-version='1' id='Subscribe1'>
|
|
<div style='white-space:nowrap'>
|
|
<h2 class='title'>
|
|
Subscribe
|
|
</h2>
|
|
<div class='widget-content'>
|
|
<div class='subscribe-wrapper subscribe-type-POST'>
|
|
<div class='subscribe expanded subscribe-type-POST' id='SW_READER_LIST_Subscribe1POST' style='display:none;'>
|
|
<div class='top'>
|
|
<span class='inner' onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
|
|
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
|
|
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
|
|
Posts
|
|
</span>
|
|
<div class='feed-reader-links'>
|
|
<a class='feed-reader-link' href='http://www.netvibes.com/subscribe.php?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='_blank'>
|
|
<img src='https://resources.blogblog.com/img/widgets/subscribe-netvibes.png'/>
|
|
</a>
|
|
<a class='feed-reader-link' href='http://add.my.yahoo.com/content?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='_blank'>
|
|
<img src='https://resources.blogblog.com/img/widgets/subscribe-yahoo.png'/>
|
|
</a>
|
|
<a class='feed-reader-link' href='https://sysadvent.blogspot.com/feeds/posts/default' target='_blank'>
|
|
<img align='absmiddle' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
|
|
Atom
|
|
</a>
|
|
</div>
|
|
</div>
|
|
<div class='bottom'></div>
|
|
</div>
|
|
<div class='subscribe' id='SW_READER_LIST_CLOSED_Subscribe1POST' onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
|
|
<div class='top'>
|
|
<span class='inner'>
|
|
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
|
|
<span onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
|
|
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
|
|
Posts
|
|
</span>
|
|
</span>
|
|
</div>
|
|
<div class='bottom'></div>
|
|
</div>
|
|
</div>
|
|
<div class='subscribe-wrapper subscribe-type-PER_POST'>
|
|
<div class='subscribe expanded subscribe-type-PER_POST' id='SW_READER_LIST_Subscribe1PER_POST' style='display:none;'>
|
|
<div class='top'>
|
|
<span class='inner' onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
|
|
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
|
|
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
|
|
Comments
|
|
</span>
|
|
<div class='feed-reader-links'>
|
|
<a class='feed-reader-link' href='http://www.netvibes.com/subscribe.php?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2F7842984104311049436%2Fcomments%2Fdefault' target='_blank'>
|
|
<img src='https://resources.blogblog.com/img/widgets/subscribe-netvibes.png'/>
|
|
</a>
|
|
<a class='feed-reader-link' href='http://add.my.yahoo.com/content?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2F7842984104311049436%2Fcomments%2Fdefault' target='_blank'>
|
|
<img src='https://resources.blogblog.com/img/widgets/subscribe-yahoo.png'/>
|
|
</a>
|
|
<a class='feed-reader-link' href='https://sysadvent.blogspot.com/feeds/7842984104311049436/comments/default' target='_blank'>
|
|
<img align='absmiddle' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
|
|
Atom
|
|
</a>
|
|
</div>
|
|
</div>
|
|
<div class='bottom'></div>
|
|
</div>
|
|
<div class='subscribe' id='SW_READER_LIST_CLOSED_Subscribe1PER_POST' onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
|
|
<div class='top'>
|
|
<span class='inner'>
|
|
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
|
|
<span onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
|
|
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
|
|
Comments
|
|
</span>
|
|
</span>
|
|
</div>
|
|
<div class='bottom'></div>
|
|
</div>
|
|
</div>
|
|
<div style='clear:both'></div>
|
|
</div>
|
|
</div>
|
|
<div class='clear'></div>
|
|
</div><div class='widget BlogArchive' data-version='1' id='BlogArchive2'>
|
|
<h2>
|
|
Blog Archive
|
|
</h2>
|
|
<div class='widget-content'>
|
|
<div id='ArchiveList'>
|
|
<div id='BlogArchive2_ArchiveList'>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2021/'>
|
|
2021
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
22
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2021/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
22
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2019/'>
|
|
2019
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2019/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate expanded'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy toggle-open'>
|
|
▼ 
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2017/'>
|
|
2017
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate expanded'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy toggle-open'>
|
|
▼ 
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2017/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='posts'>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-25-how-to-choose-data-store-for-new.html'>
|
|
Day 25 - How to choose a data store for the new sh...
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-24-on-premise-kubernetes-with.html'>
|
|
Day 24 - On-premise Kubernetes with dynamic load b...
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-23-open-source-licensing-in-real.html'>
|
|
Day 23 - Open Source Licensing in the Real World
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-22-building-secure-bastion-host-or.html'>
|
|
Day 22 - Building a secure bastion host, or, 50 wa...
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-21-lighting-up-your-haunted.html'>
|
|
Day 21 - Lighting Up Your Haunted Graveyards
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-20-regarding-responsibility-of.html'>
|
|
Day 20 - Regarding the Responsibility of Systems A...
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-19-infrastructure-testing-sanity.html'>
|
|
Day 19 - Infrastructure Testing: Sanity Checking V...
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-18-awesome-command-line-fuzzy.html'>
|
|
Day 18 - Awesome command-line fuzzy finding with fzf
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-17-don-fall-for-hybrid-cloud-trap.html'>
|
|
Day 17 - Don’t Fall for the Hybrid Cloud Trap
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-16-inspec-gives-insight.html'>
|
|
Day 16 - Inspec gives insight
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-15-devops-christmas-carol.html'>
|
|
Day 15 - A DevOps Christmas Carol
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-14-pets-vs-cattle-prods-silence-of.html'>
|
|
Day 14 - Pets vs. Cattle Prods: The Silence of the...
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-13-half-dead-tcp-connections-and.html'>
|
|
Day 13 - Half-Dead TCP Connections and Why Heartbe...
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-12-monitoring-postgres-replication.html'>
|
|
Day 12 - Monitoring Postgres Replication Lag
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-11-scaling-your-on-duty-team.html'>
|
|
Day 11 - Scaling your on-duty team
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-10-from-product-eng-to-systems-eng.html'>
|
|
Day 10 - From Product Eng to Systems Eng
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-9-using-kubernetes-for-multi.html'>
|
|
Day 9 - Using Kubernetes for multi-provider, multi...
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-8-breaking-in-new-company-as-sre.html'>
|
|
Day 8 - Breaking in a New Company as an SRE
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-7-running-inspec-as-push-job.html'>
|
|
Day 7 - Running InSpec as a Push Job…or…The Nightm...
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-6-sysadmins-evolution-of-role.html'>
|
|
Day 6 - sysadmins - the evolution of a role amidst...
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-5-do-you-want-to-build-helm-chart.html'>
|
|
Day 5 - Do you want to build a helm chart?
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-4-this-christmas-build-your-own-pki.html'>
|
|
Day 4 - This Christmas, build your own PKI
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-3-root-cause-is-plural.html'>
|
|
Day 3 - Root Cause is Plural
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-2-shifting-left-securely-with-inspec.html'>
|
|
Day 2 - Shifting Left Securely With inSpec
|
|
</a>
|
|
</li>
|
|
<li>
|
|
<a href='https://sysadvent.blogspot.com/2017/12/day-1-comfortably-numb-or-corporate.html'>
|
|
Day 1 - Comfortably Numb - or - “How Corporate Ana...
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2016/'>
|
|
2016
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2016/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2015/'>
|
|
2015
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2015/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2014/'>
|
|
2014
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2014/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2013/'>
|
|
2013
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2013/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2012/'>
|
|
2012
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2012/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2011/'>
|
|
2011
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2011/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2010/'>
|
|
2010
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2010/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2009/'>
|
|
2009
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
26
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2009/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
26
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2008/'>
|
|
2008
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
<ul class='hierarchy'>
|
|
<li class='archivedate collapsed'>
|
|
<a class='toggle' href='javascript:void(0)'>
|
|
<span class='zippy'>
|
|
|
|
► 
|
|
|
|
</span>
|
|
</a>
|
|
<a class='post-count-link' href='https://sysadvent.blogspot.com/2008/12/'>
|
|
December
|
|
</a>
|
|
<span class='post-count' dir='ltr'>
|
|
(
|
|
25
|
|
)
|
|
</span>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
<div class='clear'></div>
|
|
</div>
|
|
</div></div>
|
|
</div>
|
|
<!-- spacer for skins that want sidebar and main to be the same height-->
|
|
<div class='clear'>
|
|
 
|
|
</div>
|
|
</div>
|
|
<!-- end content-wrapper -->
|
|
<div id='footer-wrapper'>
|
|
<div class='footer no-items section' id='footer'></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<!-- end outer-wrapper -->
|
|
<script type='text/javascript'>
|
|
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
|
|
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
|
|
</script>
|
|
<script type='text/javascript'>
|
|
try {
|
|
var pageTracker = _gat._getTracker("UA-6522917-1");
|
|
pageTracker._trackPageview();
|
|
} catch(err) {}</script>
|
|
|
|
<script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/851759228-widgets.js"></script>
|
|
<script type='text/javascript'>
|
|
window['__wavt'] = 'AAPvtVlTN2pu1LRbYiT-80pGy3G1:1789096562644';_WidgetManager._Init('//draft.blogger.com/rearrange?blogID\x3d3615332969083650973','//sysadvent.blogspot.com/2017/12/day-3-root-cause-is-plural.html','3615332969083650973');
|
|
_WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '3615332969083650973', 'title': 'sysadvent', 'url': 'https://sysadvent.blogspot.com/2017/12/day-3-root-cause-is-plural.html', 'canonicalUrl': 'https://sysadvent.blogspot.com/2017/12/day-3-root-cause-is-plural.html', 'homepageUrl': 'https://sysadvent.blogspot.com/', 'searchUrl': 'https://sysadvent.blogspot.com/search', 'canonicalHomepageUrl': 'https://sysadvent.blogspot.com/', 'blogspotFaviconUrl': 'https://sysadvent.blogspot.com/favicon.ico', 'bloggerUrl': 'https://draft.blogger.com', 'hasCustomDomain': false, 'httpsEnabled': true, 'enabledCommentProfileImages': false, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': '', 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22sysadvent - RSS\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://draft.blogger.com/feeds/3615332969083650973/posts/default\x22 /\x3e\n\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/7842984104311049436/comments/default\x22 /\x3e\n', 'meTag': '', 'adsenseClientId': 'ca-pub-6194074709963145', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'isGoogleEverywhereLinkTooltipEnabled': true, 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/31d4351e075118ef', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'item', 'postId': '7842984104311049436', 'postImageThumbnailUrl': 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEivpMyPUQ-v2cJEUNgoFPVnIaynlOziDn-07g6XfgojLv-wA8RW5PApntTuP7yK6g6nn1Hd4ykMn-FUSNF_WSiR9ZW32j03qFrdKFl8Qkq5Yj6fuPVX4nvl7rIsVWwnvbSTfskN3OYh1dU/s72-c/cause-effect-tree.png', 'postImageUrl': 'https://upload.wikimedia.org/wikipedia/commons/0/04/Cycads_root.png', 'pageName': 'Day 3 - Root Cause is Plural', 'pageTitle': 'sysadvent: Day 3 - Root Cause is Plural'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Day 3 - Root Cause is Plural', 'description': 'Written by Cody Wilbourn ( @codywilbourn ) Edited by Kerim Satirli ( @ksatirli ) Post-mortems are an industry standard process that happe...', 'featuredImage': 'https://lh3.googleusercontent.com/blogger_img_proxy/AEn0k_vt_1WbGcO-eXmNGsdqNIIxPzwO9CWQOOt4tw4WjWDiCJm9g41mkwmulfkiXCSSk5ZzspYsTuoXZhwCHvm6tFC9TkdhdzAYqXxOI1HrlnSurkMvXuFoUQM04eAIsvVdRpYyRrcOafc', 'url': 'https://sysadvent.blogspot.com/2017/12/day-3-root-cause-is-plural.html', 'type': 'item', 'isSingleItem': true, 'isMultipleItems': false, 'isError': false, 'isPage': false, 'isPost': true, 'isHomepage': false, 'isArchive': false, 'isLabelSearch': false, 'postId': 7842984104311049436}}]);
|
|
_WidgetManager._RegisterWidget('_NavbarView', new _WidgetInfo('Navbar1', 'navbar', document.getElementById('Navbar1'), {}, 'displayModeFull'));
|
|
_WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull'));
|
|
_WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/120969204-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/828616780-lightbox_bundle.css'}, 'displayModeFull'));
|
|
_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML2', 'sidebar', document.getElementById('HTML2'), {}, 'displayModeFull'));
|
|
_WidgetManager._RegisterWidget('_SubscribeView', new _WidgetInfo('Subscribe1', 'sidebar', document.getElementById('Subscribe1'), {}, 'displayModeFull'));
|
|
_WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive2', 'sidebar', document.getElementById('BlogArchive2'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\x26hellip;'}, 'displayModeFull'));
|
|
</script>
|
|
</body>
|
|
</html> |