Files
nexus/sreweekly/articles/303/07-day-22-so-you-re-incident-commander-now-what.html
2026-09-12 17:23:01 +08:00

1473 lines
54 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html dir='ltr' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'>
<head>
<link href='https://www.blogger.com/static/v1/widgets/4165133002-widget_css_bundle.css' rel='stylesheet' type='text/css'/>
<meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/>
<meta content='blogger' name='generator'/>
<link href='https://sysadvent.blogspot.com/favicon.ico' rel='icon' type='image/x-icon'/>
<link href='https://sysadvent.blogspot.com/2021/12/day-22-so-youre-incident-commander-now.html' rel='canonical'/>
<link rel="alternate" type="application/atom+xml" title="sysadvent - Atom" href="https://sysadvent.blogspot.com/feeds/posts/default" />
<link rel="alternate" type="application/rss+xml" title="sysadvent - RSS" href="https://sysadvent.blogspot.com/feeds/posts/default?alt=rss" />
<link rel="service.post" type="application/atom+xml" title="sysadvent - Atom" href="https://www.blogger.com/feeds/3615332969083650973/posts/default" />
<link rel="alternate" type="application/atom+xml" title="sysadvent - Atom" href="https://sysadvent.blogspot.com/feeds/9028706749643882414/comments/default" />
<!--Can't find substitution for tag [blog.ieCssRetrofitLinks]-->
<meta content='https://sysadvent.blogspot.com/2021/12/day-22-so-youre-incident-commander-now.html' property='og:url'/>
<meta content='Day 22 - So, You&#39;re Incident Commander, Now What?' property='og:title'/>
<meta content=' By: Joshua Timberman ( @jtimberman ) You’re the SRE on call and, while working on a project, your phone buzzes with an alert: “Eleva...' property='og:description'/>
<title>
sysadvent: Day 22 - So, You're Incident Commander, Now What?
</title>
<style type='text/css'>@font-face{font-family:'Allerta';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/allerta/v19/TwMO-IAHRlkbx940YnYXSCiN9uc.woff2)format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDRrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcBBrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDBrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+1F00-1FFF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcAxrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0370-0377,U+037A-037F,U+0384-038A,U+038C,U+038E-03A1,U+03A3-03FF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcAhrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0307-0308,U+0590-05FF,U+200C-2010,U+20AA,U+25CC,U+FB1D-FB4F;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDxrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDhrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcABrBdwcoaaQw.woff2)format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;}</style>
<style id='page-skin-1' type='text/css'><!--
/*
-----------------------------------------------
Blogger Template Style
Name: Minima
Designer: Douglas Bowman
URL: www.stopdesign.com
Date: 26 Feb 2004
Updated by: Blogger Team
----------------------------------------------- */
/* Variable definitions
====================
<Variable name="bgcolor" description="Page Background Color"
type="color" default="#fff">
<Variable name="textcolor" description="Text Color"
type="color" default="#333">
<Variable name="linkcolor" description="Link Color"
type="color" default="#58a">
<Variable name="pagetitlecolor" description="Blog Title Color"
type="color" default="#666">
<Variable name="descriptioncolor" description="Blog Description Color"
type="color" default="#999">
<Variable name="titlecolor" description="Post Title Color"
type="color" default="#c60">
<Variable name="bordercolor" description="Border Color"
type="color" default="#ccc">
<Variable name="sidebarcolor" description="Sidebar Title Color"
type="color" default="#999">
<Variable name="sidebartextcolor" description="Sidebar Text Color"
type="color" default="#666">
<Variable name="visitedlinkcolor" description="Visited Link Color"
type="color" default="#999">
<Variable name="bodyfont" description="Text Font"
type="font" default="normal normal 100% Georgia, Serif">
<Variable name="headerfont" description="Sidebar Title Font"
type="font"
default="normal normal 78% 'Trebuchet MS',Trebuchet,Arial,Verdana,Sans-serif">
<Variable name="pagetitlefont" description="Blog Title Font"
type="font"
default="normal normal 200% Georgia, Serif">
<Variable name="descriptionfont" description="Blog Description Font"
type="font"
default="normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif">
<Variable name="postfooterfont" description="Post Footer Font"
type="font"
default="normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif">
<Variable name="startSide" description="Side where text starts in blog language"
type="automatic" default="left">
<Variable name="endSide" description="Side where text ends in blog language"
type="automatic" default="right">
*/
/* Use this with templates/template-twocol.html */
body {
background:#ffffff;
margin:0;
color:#333333;
font:x-small Georgia Serif;
font-size/* */:/**/small;
font-size: /**/small;
text-align: center;
}
a:link {
color:#5588aa;
text-decoration:none;
}
a:visited {
color:#999999;
text-decoration:none;
}
a:hover {
color:#cc6600;
text-decoration:underline;
}
a img {
border-width:0;
}
/* Header
-----------------------------------------------
*/
#header-wrapper {
width:760px;
margin:0 auto 10px;
/*border:1px solid #cccccc;*/
}
#header-inner {
background-position: center;
margin-left: auto;
margin-right: auto;
}
#header {
margin: 5px;
/*border: 1px solid #cccccc;*/
text-align: center;
color:#666666;
}
#header h1 {
margin:5px 5px 0;
padding:15px 20px .25em;
line-height:1.2em;
text-transform:uppercase;
letter-spacing:.2em;
font: normal normal 200% Georgia, Serif;
}
#header a {
color:#666666;
text-decoration:none;
}
#header a:hover {
color:#666666;
}
#header .description {
margin:0 5px 5px;
padding:0 20px 15px;
max-width:800px;
text-transform:uppercase;
letter-spacing:.2em;
line-height: 1.4em;
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
color: #999999;
}
#header img {
margin-left: auto;
margin-right: auto;
}
/* Outer-Wrapper
----------------------------------------------- */
#outer-wrapper {
width: 760px;
margin:0 auto;
padding:10px;
text-align:left;
font: normal normal 100% Arimo;
}
#main-wrapper {
width: 510px;
float: left;
word-wrap: break-word; /* fix for long text breaking sidebar float in IE */
overflow: hidden; /* fix for long non-text content breaking IE sidebar float */
}
#sidebar-wrapper {
width: 220px;
float: right;
word-wrap: break-word; /* fix for long text breaking sidebar float in IE */
overflow: hidden; /* fix for long non-text content breaking IE sidebar float */
}
/* Headings
----------------------------------------------- */
h2 {
margin:1.5em 0 .75em;
font:normal normal 130% Allerta;
line-height: 1.4em;
text-transform:uppercase;
letter-spacing:.2em;
color:#999999;
}
/* Posts
-----------------------------------------------
*/
h2.date-header {
margin:1.5em 0 .5em;
}
.post {
margin:.5em 0 1.5em;
border-bottom:1px dotted #cccccc;
padding-bottom:1.5em;
}
.post h3 {
margin:.25em 0 0;
padding:0 0 4px;
font-size:140%;
font-weight:normal;
line-height:1.4em;
color:#cc6600;
}
.post h3 a, .post h3 a:visited, .post h3 strong {
display:block;
text-decoration:none;
color:#cc6600;
font-weight:normal;
}
.post h3 strong, .post h3 a:hover {
color:#333333;
}
.post-body {
margin:0 0 .75em;
line-height:1.6em;
}
.post-body blockquote {
line-height:1.3em;
}
.post-footer {
margin: .75em 0;
color:#999999;
text-transform:uppercase;
letter-spacing:.1em;
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
line-height: 1.4em;
}
.comment-link {
margin-left:.6em;
}
.post img {
padding:4px;
border:1px solid #cccccc;
}
.post blockquote {
margin:1em 20px;
}
.post blockquote p {
margin:.75em 0;
}
.post pre {
overflow: auto;
padding-left: 5px;
padding-right: 5px;
border: 1px solid #CCCCFF !important;
}
.post pre a {
text-decoration: underline;
}
/* Comments
----------------------------------------------- */
#comments h4 {
margin:1em 0;
font-weight: bold;
line-height: 1.4em;
text-transform:uppercase;
letter-spacing:.2em;
color: #999999;
}
#comments-block {
margin:1em 0 1.5em;
line-height:1.6em;
}
#comments-block .comment-author {
margin:.5em 0;
}
#comments-block .comment-body {
margin:.25em 0 0;
}
#comments-block .comment-footer {
margin:-.25em 0 2em;
line-height: 1.4em;
text-transform:uppercase;
letter-spacing:.1em;
}
#comments-block .comment-body p {
margin:0 0 .75em;
}
.deleted-comment {
font-style:italic;
color:gray;
}
#blog-pager-newer-link {
float: left;
}
#blog-pager-older-link {
float: right;
}
#blog-pager {
text-align: center;
}
.feed-links {
clear: both;
line-height: 2.5em;
}
/* Sidebar Content
----------------------------------------------- */
.sidebar {
color: #666666;
line-height: 1.5em;
}
.sidebar ul {
list-style:none;
margin:0 0 0;
padding:0 0 0;
}
.sidebar li {
margin:0;
padding-top:0;
padding-right:0;
padding-bottom:.25em;
padding-left:15px;
text-indent:-15px;
line-height:1.5em;
}
.sidebar .widget, .main .widget {
border-bottom:1px dotted #cccccc;
margin:0 0 1.5em;
padding:0 0 1.5em;
}
.main .Blog {
border-bottom-width: 0;
}
/* Profile
----------------------------------------------- */
.profile-img {
float: left;
margin-top: 0;
margin-right: 5px;
margin-bottom: 5px;
margin-left: 0;
padding: 4px;
border: 1px solid #cccccc;
}
.profile-data {
margin:0;
text-transform:uppercase;
letter-spacing:.1em;
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
color: #999999;
font-weight: bold;
line-height: 1.6em;
}
.profile-datablock {
margin:.5em 0 .5em;
}
.profile-textblock {
margin: 0.5em 0;
line-height: 1.6em;
}
.profile-link {
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
text-transform: uppercase;
letter-spacing: .1em;
}
/* Footer
----------------------------------------------- */
#footer {
width:760px;
clear:both;
margin:0 auto;
padding-top:15px;
line-height: 1.6em;
text-transform:uppercase;
letter-spacing:.1em;
text-align: center;
}
@media only screen and (max-width:480px) {
a:link {
color: #d12a39;
}
.mobile-link-button {
background-color: #d12a39;
}
.mobile-link-button a:link, .mobile-link-button a:visited {
color: #ffffff;
}
}
--></style>
<link href='https://www.blogger.com/dyncss/3615332969083650973/authorization.css?zx=24cd04b2-38ff-45f4-a3f0-f0a07f75ec8f' media='none' onload='if(media!=&#39;all&#39;)media=&#39;all&#39;' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyncss/3615332969083650973/authorization.css?zx=24cd04b2-38ff-45f4-a3f0-f0a07f75ec8f' rel='stylesheet'/></noscript>
<meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/>
<meta name='google-adsense-platform-domain' content='blogspot.com'/>
<!-- data-ad-client=ca-pub-6194074709963145 -->
</head>
<body>
<div class='navbar section' id='navbar'><div class='widget Navbar' data-version='1' id='Navbar1'><script type="text/javascript">
function setAttributeOnload(object, attribute, val) {
if(window.addEventListener) {
window.addEventListener('load',
function(){ object[attribute] = val; }, false);
} else {
window.attachEvent('onload', function(){ object[attribute] = val; });
}
}
</script>
<div id="navbar-iframe-container"></div>
<script type="text/javascript" src="https://apis.google.com/js/platform.js"></script>
<script type="text/javascript">
gapi.load("gapi.iframes:gapi.iframes.style.bubble", function() {
if (gapi.iframes && gapi.iframes.getContext) {
gapi.iframes.getContext().openChild({
url: 'https://www.blogger.com/navbar/3615332969083650973?po\x3d9028706749643882414\x26origin\x3dhttps://sysadvent.blogspot.com',
where: document.getElementById("navbar-iframe-container"),
id: "navbar-iframe"
});
}
});
</script><script type="text/javascript">
(function() {
var script = document.createElement('script');
script.type = 'text/javascript';
script.src = '//pagead2.googlesyndication.com/pagead/js/google_top_exp.js';
var head = document.getElementsByTagName('head')[0];
if (head) {
head.appendChild(script);
}})();
</script>
</div></div>
<div id='outer-wrapper'>
<div id='wrap2'>
<!-- skip links for text browsers -->
<span id='skiplinks' style='display:none;'>
<a href='#main'>
skip to main
</a>
|
<a href='#sidebar'>
skip to sidebar
</a>
</span>
<div id='header-wrapper'>
<div class='header section' id='header'><div class='widget Header' data-version='1' id='Header1'>
<div id='header-inner'>
<a href='https://sysadvent.blogspot.com/' style='display: block'>
<img alt='sysadvent' height='240px; ' id='Header1_headerimg' src='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiYVlIMEU0sXvTBtcY5iFj5QtiT3Klf0S3sgSamkTPsjU1QGvOWHA-6yq173iHys3-ImrIFY26KIPuKEoSNSbitvtnXsZU6b8YPGQ48ir-8Jo2NFrTsaiO_Nl9W9XLt_wyWhTM_16_HeN0/s750/sys-advent-long.png' style='display: block' width='750px; '/>
</a>
</div>
</div></div>
</div>
<div id='content-wrapper'>
<div id='crosscol-wrapper' style='text-align:center'>
<div class='crosscol no-items section' id='crosscol'></div>
</div>
<div id='main-wrapper'>
<div class='main section' id='main'><div class='widget Blog' data-version='1' id='Blog1'>
<div class='blog-posts hfeed'>
<!--Can't find substitution for tag [defaultAdStart]-->
<div class="date-outer">
<h2 class='date-header'>
<span>
December 22, 2021
</span>
</h2>
<div class="date-posts">
<div class='post-outer'>
<div class='post hentry'>
<a name='9028706749643882414'></a>
<h3 class='post-title entry-title'>
<a href='https://sysadvent.blogspot.com/2021/12/day-22-so-youre-incident-commander-now.html'>
Day 22 - So, You're Incident Commander, Now What?
</a>
</h3>
<div class='post-header-line-1'></div>
<div class='post-body entry-content'>
<p>
By: Joshua Timberman (<a href="https://twitter.com/jtimberman">@jtimberman</a>)<br />
</p>
<p>
You&#8217;re the SRE on call and, while working on a project, your phone buzzes with
an alert: &#8220;Elevated 500s from API.&#8221;
</p>
<p>
You&#8217;re a software developer, and your team lead posts in Slack: &#8220;Hey, the
library we use for our payment processing endpoint has a remote exploit.&#8221;
</p>
<p>
You work on the customer success team and, during a routine sync with a
high-profile customer, they install the new version of your client CLI. Then,
when they run <em>any</em> command, it exits with a non-zero return code.
</p>
<p>
An incident is <em>any</em> situation that disrupts the ability of customers to
use a system, service, or software product in a safe and secure manner. And in
each of the incidents above, the person who noticed the incident first will most
likely become the incident commander. So, now what?
</p>
<h2>What does it mean to be an incident commander?</h2>
<p>
Once an individual identifies an incident, one or more people will respond to
it. Their goal is to resolve the incident and return systems, services, or other
software back to a functional state. While an incident may have a few or many
responders&#8212;only one person is the incident commander. This role is not about
experience, seniority, or position on an org chart; it is to ensure that
progress is being made to resolve the incident. The incident commander must
think about the inputs from the incident, make decisions about what to do next,
and communicate with others about what is happening. The incident commander also
determines when the incident is resolved based on the information they have.
After the incident is over, the incident commander is also responsible for
conducting a post-incident analysis and review to summarize what happened, what
the team learned, and what will be done to mitigate the risk of a similar
incident happening in the future.
</p>
<p>
Having a single person&#8212;the incident commander&#8212;be responsible for handling the
incident, delegating responsibility to others, determining when the incident is
resolved, and conducting the post-incident review is one of the most effective
incident management strategies.
</p>
<h2>How do you become an incident commander?</h2>
<p>
Organizations vary on how a team member can become an incident commander. Some
call upon the first responder to an incident. Others require specific training
and have an on-call rotation of just incident commanders. However you find
yourself in the role of incident commander, you should be trusted and empowered
by your organization to lead the effort to resolve the incident.
</p>
<h2>Now what?</h2>
<p>
Now that you&#8217;re incident commander, follow your organizations&#8217; incident response
procedure for the specifics about what to do. But for more general questions,
we&#8217;ve got some guidance.
</p>
<h3>What are the best strategies for communication and coordination?</h3>
<p>
One of an incident commander&#8217;s primary tasks is to communicate with relevant
teams and stakeholders about the status of the incident and to coordinate with
other teams to ensure the right people are involved.
</p>
<p>
If your <a href="https://allma.io/blog/effective-incident-management-using-slack">primary
communication tool is Slack</a>, use a separate channel for each incident.
Prefix any time-sensitive notes with &#8220;timeline&#8221; or &#8220;TL&#8221; so they are easy to find
later. If higher-bandwidth communication is required, use a video conference,
and keep the channel updated with important information and interactions. When
an incident affects external customers, be sure to update them as required by
your support teams and agreements with customers.
</p>
<p>
In the case of a security incident, there may be additional communications
requirements with your organization&#8217;s legal and/or marketing teams. Legal
considerations to communicate may include:
</p>
<ul>
<li>Statutory or regulatory reporting
<li>Contractual commitments and obligations to customers
<li>Insurance claims
</li>
</ul>
<p>
Marketing considerations to communicate may include:
</p>
<ul>
<li>Sensitive information from customer data exposure
<li>&#8220;Zero Day&#8221; exploits
<li>Special messaging requirements, e.g. for publicly traded companies
</li>
</ul>
<h3>When should you hand off a long-running incident?</h3>
<p>
During an extended outage or other long-running incident, you will likely need a
break. Whether you are feeling overwhelmed, or that you would contribute better
by working on a solution for the incident itself, or that you need to eat, take
care of your family, or sleep&#8212;all are good reasons to hand off the incident
command to someone else.
</p>
<p>
Coordinate with your other responders in the appropriate channel, whether that&#8217;s
in a Slack chat or in a Zoom meeting. If necessary, escalate by having someone
paged out to get help. Once someone else can take over, communicate with them
on the latest progress, the current steps being taken, and who else is involved
with the incident. Remember, we&#8217;re all human and we need breaks.
</p>
<h3>How should you approach post-incident analysis and review?</h3>
<p>
One of an incident commander&#8217;s most important jobs is to conduct a post-incident
analysis and review after the incident is resolved. This meeting must be
<em>blameless</em>: That is, the goal of the meeting is to learn what happened,
determine what contributing factors led to the incident, and take action to
mitigate the risk of such an incident happening in the future. It&#8217;s also to
establish a timeline of events, demonstrate an understanding of the problems,
and set up the organization for future success in mitigating that problem.
</p>
<p>
The sooner the incident analysis and review meeting occurs after the incident is
resolved, the better. You should ensure adequate rest time for yourself and
other responders, but the review meeting should happen within 24 hours&#8212;and
ideally not longer than two business days after the incident. The incident
commander (or commanders) must attend, as they have the most context on what
happened and what decisions were made. Any responders who performed significant
remediation steps or investigation must also attend so they can share what they
learned and what they did during the incident.
</p>
<p>
Because the systems that fail and cause incidents are complex, a good analysis
and review process is complex. Let&#8217;s break it down:
</p>
<h4>Describe the incident</h4>
<p>
The incident commander will describe the incident. This description should
detail the impact as well as its scope, i.e., whether the incident affected
internal or external users, how long it took to discover, how long it took to
recover, and what major steps were taken to resolve the incident.
</p>
<p>
&#8220;The platform was down&#8221; is not a good description.
</p>
<p>
&#8220;On its 5 minute check interval, our monitoring system alerted the on-call
engineer that the API service was non-responsive, which meant external customers
could not run their workflows for 15 minutes until we were able to restart the
message queue&#8221; is a good description.
</p>
<h4>Contributing factors</h4>
<p>
Successful incident analysis should identify the contributing factors and places
where improvements can be made to systems and software. Our world is complex,
and technology stacks have multiple moving parts and places where failures
occur. Not only can a contributing factor be something technical like &#8220;a
configuration change was made to an application,&#8221; it can be nontechnical like
&#8220;the organization didn&#8217;t budget for new hardware to improve performance.&#8221; In
reviewing the incident for contributing factors, incident commanders and
responders are looking for areas for improvement in order to identify potential
corrective actions.
</p>
<h4>Corrective action items</h4>
<p>
Finally, incident analysis should determine corrective action items. These must
be specific work items that are assigned to a person or a team accountable for
their completion, and they must be the primary work priority for that person or
team. These aren&#8217;t &#8220;nice to have,&#8221; these are &#8220;must do to ensure the safe and
reliable operation of the site or service.&#8221; Such tasks aren&#8217;t necessarily the
actions taken during the incident to stabilize or remediate a problem, which are
often temporary workarounds to restore service. A corrective action can be as
simple as adding new monitoring alerts or system metrics that weren&#8217;t
implemented before. It can also be as complex as rebuilding a database cluster
with a different high availability strategy or migrating to a different database
service entirely.
</p>
<h2>Conclusion</h2>
<p>
If you&#8217;ve recently been the incident commander for your first
incident&#8212;congratulations. You&#8217;ve worked to solve a hard problem that had a lot
of moving parts. You took on the role and communicated with the relevant teams
and stakeholders. Then, you got some much needed rest and conducted a successful
post-incident analysis. Your team identified corrective actions, and your site
or service is going to be more reliable for your customers.
</p>
<p>
Incident management is one of the most stressful aspects of operations work for
DevOps and SRE professionals. The first time you become an incident commander,
it may be confusing or upsetting. Don&#8217;t panic. You&#8217;re doing just fine, and
you&#8217;ll keep getting better.
</p>
<h2>Further reading</h2>
<p>
If you&#8217;re new to post incident analysis and review, check out <a href="https://www.jeli.io/howie-the-post-incident-guide/">Howie: The
Post-Incident Guide</a> from Jeli.
</p>
<p>
PagerDuty also has extensive documentation on <a href="https://response.pagerduty.com/">incident response</a> and <a href="https://response.pagerduty.com/training/incident_commander/">incident
command</a>.
</p>
<div style='clear: both;'></div>
</div>
<div class='post-footer'>
<div class='post-footer-line post-footer-line-1'>
<span class='post-author vcard'>
Posted by
<span class='fn'>
sigje
</span>
</span>
<span class='post-timestamp'>
</span>
<span class='post-comment-link'>
</span>
<span class='post-icons'>
<span class='item-action'>
<a href='https://www.blogger.com/email-post/3615332969083650973/9028706749643882414' title='Email Post'>
<img alt="" class="icon-action" height="13" src="//www.blogger.com/img/icon18_email.gif" width="18">
</a>
</span>
<span class='item-control blog-admin pid-1271197227'>
<a href='https://www.blogger.com/post-edit.g?blogID=3615332969083650973&postID=9028706749643882414&from=pencil' title='Edit Post'>
<img alt="" class="icon-action" height="18" src="//img2.blogblog.com/img/icon18_edit_allbkg.gif" width="18">
</a>
</span>
</span>
</div>
<div class='post-footer-line post-footer-line-2'>
<span class='post-labels'>
Labels:
<a href='https://sysadvent.blogspot.com/search/label/incident%20commander' rel='tag'>
incident commander
</a>
,
<a href='https://sysadvent.blogspot.com/search/label/incident%20management' rel='tag'>
incident management
</a>
</span>
</div>
<div class='post-footer-line post-footer-line-3'></div>
</div>
</div>
<div class='comments' id='comments'>
<a name='comments'></a>
<h4>
No comments
:
</h4>
<div id='Blog1_comments-block-wrapper'>
<dl class='' id='comments-block'>
</dl>
</div>
<p class='comment-footer'>
<a href='https://www.blogger.com/comment/fullpage/post/3615332969083650973/9028706749643882414' onclick=''>
Post a Comment
</a>
</p>
<div id='backlinks-container'>
<div id='Blog1_backlinks-container'>
</div>
</div>
</div>
</div>
</div></div>
<!--Can't find substitution for tag [adEnd]-->
</div>
<div class='blog-pager' id='blog-pager'>
<span id='blog-pager-newer-link'>
<a class='blog-pager-newer-link' href='https://sysadvent.blogspot.com/2021/12/day-23-what-is-ebpf.html' id='Blog1_blog-pager-newer-link' title='Newer Post'>
Newer Post
</a>
</span>
<span id='blog-pager-older-link'>
<a class='blog-pager-older-link' href='https://sysadvent.blogspot.com/2021/12/day-20-to-deploy-or-not-to-deploy-that.html' id='Blog1_blog-pager-older-link' title='Older Post'>
Older Post
</a>
</span>
<a class='home-link' href='https://sysadvent.blogspot.com/'>
Home
</a>
</div>
<div class='clear'></div>
<div class='post-feeds'>
<div class='feed-links'>
Subscribe to:
<a class='feed-link' href='https://sysadvent.blogspot.com/feeds/9028706749643882414/comments/default' target='_blank' type='application/atom+xml'>
Post Comments
(
Atom
)
</a>
</div>
</div>
</div></div>
</div>
<div id='sidebar-wrapper'>
<div class='sidebar section' id='sidebar'><div class='widget HTML' data-version='1' id='HTML2'>
<h2 class='title'>
What is sysadvent?
</h2>
<div class='widget-content'>
<p>
One article for each day of December, ending on the 25th article.
</p><p>
With the goals of sharing, openness, and mentoring, we aim to provide great articles about systems administration topics written by fellow sysadmins.
</p><p>
Want to get involved? Join the <a href="http://groups.google.com/group/sysadvent">mailing list</a>!
<p></p></p>
</div>
<div class='clear'></div>
</div><div class='widget Subscribe' data-version='1' id='Subscribe1'>
<div style='white-space:nowrap'>
<h2 class='title'>
Subscribe
</h2>
<div class='widget-content'>
<div class='subscribe-wrapper subscribe-type-POST'>
<div class='subscribe expanded subscribe-type-POST' id='SW_READER_LIST_Subscribe1POST' style='display:none;'>
<div class='top'>
<span class='inner' onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Posts
</span>
<div class='feed-reader-links'>
<a class='feed-reader-link' href='http://www.netvibes.com/subscribe.php?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-netvibes.png'/>
</a>
<a class='feed-reader-link' href='http://add.my.yahoo.com/content?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-yahoo.png'/>
</a>
<a class='feed-reader-link' href='https://sysadvent.blogspot.com/feeds/posts/default' target='_blank'>
<img align='absmiddle' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Atom
</a>
</div>
</div>
<div class='bottom'></div>
</div>
<div class='subscribe' id='SW_READER_LIST_CLOSED_Subscribe1POST' onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
<div class='top'>
<span class='inner'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<span onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Posts
</span>
</span>
</div>
<div class='bottom'></div>
</div>
</div>
<div class='subscribe-wrapper subscribe-type-PER_POST'>
<div class='subscribe expanded subscribe-type-PER_POST' id='SW_READER_LIST_Subscribe1PER_POST' style='display:none;'>
<div class='top'>
<span class='inner' onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Comments
</span>
<div class='feed-reader-links'>
<a class='feed-reader-link' href='http://www.netvibes.com/subscribe.php?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2F9028706749643882414%2Fcomments%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-netvibes.png'/>
</a>
<a class='feed-reader-link' href='http://add.my.yahoo.com/content?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2F9028706749643882414%2Fcomments%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-yahoo.png'/>
</a>
<a class='feed-reader-link' href='https://sysadvent.blogspot.com/feeds/9028706749643882414/comments/default' target='_blank'>
<img align='absmiddle' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Atom
</a>
</div>
</div>
<div class='bottom'></div>
</div>
<div class='subscribe' id='SW_READER_LIST_CLOSED_Subscribe1PER_POST' onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
<div class='top'>
<span class='inner'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<span onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Comments
</span>
</span>
</div>
<div class='bottom'></div>
</div>
</div>
<div style='clear:both'></div>
</div>
</div>
<div class='clear'></div>
</div><div class='widget BlogArchive' data-version='1' id='BlogArchive2'>
<h2>
Blog Archive
</h2>
<div class='widget-content'>
<div id='ArchiveList'>
<div id='BlogArchive2_ArchiveList'>
<ul class='hierarchy'>
<li class='archivedate expanded'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy toggle-open'>
&#9660;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2021/'>
2021
</a>
<span class='post-count' dir='ltr'>
(
22
)
</span>
<ul class='hierarchy'>
<li class='archivedate expanded'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy toggle-open'>
&#9660;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2021/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
22
)
</span>
<ul class='posts'>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-23-what-is-ebpf.html'>
Day 23 - What is eBPF?
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-22-so-youre-incident-commander-now.html'>
Day 22 - So, You&#39;re Incident Commander, Now What?
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-20-to-deploy-or-not-to-deploy-that.html'>
Day 20 - To Deploy or Not to Deploy? That is the q...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-19-into-world-of-chaos-engineering.html'>
Day 19 - Into the World of Chaos Engineering
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-18-minimizing-false-positive.html'>
Day 18 - Minimizing False Positive Monitoring Aler...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-17-death-to-localhost-benefits-of.html'>
Day 17 - Death to Localhost: The Benefits of Devel...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-16-setting-up-k3s-in-your-home-lab.html'>
Day 16 - Setting up k3s in your home lab
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-15-introduction-to-pagerduty-api.html'>
Day 15 - Introduction to the PagerDuty API
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-14-whats-in-job-description-and-who.html'>
Day 14 - What&#39;s in a job description (and who does...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-13-ephemeral-pr-environments.html'>
Day 13 - Ephemeral PR Environments: Enabling autom...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-12-terraform-refactoring.html'>
Day 12 - Terraform Refactoring
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-11-moving-from-engineering-manager.html'>
Day 11 - Moving from Engineering Manager to IC
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-10-assembling-your-year-in-review.html'>
Day 10 - Assembling Your Year In Review
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/3-things-parenting-taught-me-about.html'>
Day 9 - 3 things parenting taught me about system ...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-8-d-for-sres.html'>
Day 8 - D&amp;D for SREs
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-7-baking-multi-architecture-docker.html'>
Day 7 - Baking Multi-architecture Docker Images
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-6-more-to-come-tomorrow.html'>
Day 6 - More to come tomorrow!
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/least-privilege-using-strace.html'>
Day 5 - Least Privilege using strace
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/gwlb-panacea-for-cloud-dmz-on-aws.html'>
Day 4 - GWLB: Panacea for Cloud DMZ on AWS
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/keeping-config-management-simple-with.html'>
Day 3 - Keeping Config Management Simple with Itamae
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/reliability-as-product-feature.html'>
Day 2 - Reliability as a Product Feature
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/the-myths-and-magic-in-my-search-for.html'>
Day 1 - The Myths and the Magic in My Search for A...
</a>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2019/'>
2019
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2019/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2017/'>
2017
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2017/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2016/'>
2016
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2016/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2015/'>
2015
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2015/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2014/'>
2014
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2014/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2013/'>
2013
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2013/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2012/'>
2012
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2012/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2011/'>
2011
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2011/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2010/'>
2010
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2010/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2009/'>
2009
</a>
<span class='post-count' dir='ltr'>
(
26
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2009/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
26
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2008/'>
2008
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2008/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
</div>
</div>
<div class='clear'></div>
</div>
</div></div>
</div>
<!-- spacer for skins that want sidebar and main to be the same height-->
<div class='clear'>
&#160;
</div>
</div>
<!-- end content-wrapper -->
<div id='footer-wrapper'>
<div class='footer no-items section' id='footer'></div>
</div>
</div>
</div>
<!-- end outer-wrapper -->
<script type='text/javascript'>
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
</script>
<script type='text/javascript'>
try {
var pageTracker = _gat._getTracker("UA-6522917-1");
pageTracker._trackPageview();
} catch(err) {}</script>
<script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/1363620594-widgets.js"></script>
<script type='text/javascript'>
window['__wavt'] = 'AAPvtVne0X_gVHVolNiTy8MDfjcm:1789189997140';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d3615332969083650973','//sysadvent.blogspot.com/2021/12/day-22-so-youre-incident-commander-now.html','3615332969083650973');
_WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '3615332969083650973', 'title': 'sysadvent', 'url': 'https://sysadvent.blogspot.com/2021/12/day-22-so-youre-incident-commander-now.html', 'canonicalUrl': 'https://sysadvent.blogspot.com/2021/12/day-22-so-youre-incident-commander-now.html', 'homepageUrl': 'https://sysadvent.blogspot.com/', 'searchUrl': 'https://sysadvent.blogspot.com/search', 'canonicalHomepageUrl': 'https://sysadvent.blogspot.com/', 'blogspotFaviconUrl': 'https://sysadvent.blogspot.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': false, 'httpsEnabled': true, 'enabledCommentProfileImages': false, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': '', 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22sysadvent - RSS\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/3615332969083650973/posts/default\x22 /\x3e\n\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/9028706749643882414/comments/default\x22 /\x3e\n', 'meTag': '', 'adsenseClientId': 'ca-pub-6194074709963145', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'isGoogleEverywhereLinkTooltipEnabled': true, 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/1890a9484e710b1c', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'item', 'postId': '9028706749643882414', 'pageName': 'Day 22 - So, You\x27re Incident Commander, Now What?', 'pageTitle': 'sysadvent: Day 22 - So, You\x27re Incident Commander, Now What?'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Day 22 - So, You\x27re Incident Commander, Now What?', 'description': ' By: Joshua Timberman ( @jtimberman ) You\u2019re the SRE on call and, while working on a project, your phone buzzes with an alert: \u201cEleva...', 'url': 'https://sysadvent.blogspot.com/2021/12/day-22-so-youre-incident-commander-now.html', 'type': 'item', 'isSingleItem': true, 'isMultipleItems': false, 'isError': false, 'isPage': false, 'isPost': true, 'isHomepage': false, 'isArchive': false, 'isLabelSearch': false, 'postId': 9028706749643882414}}]);
_WidgetManager._RegisterWidget('_NavbarView', new _WidgetInfo('Navbar1', 'navbar', document.getElementById('Navbar1'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/700075392-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/828616780-lightbox_bundle.css'}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML2', 'sidebar', document.getElementById('HTML2'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_SubscribeView', new _WidgetInfo('Subscribe1', 'sidebar', document.getElementById('Subscribe1'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive2', 'sidebar', document.getElementById('BlogArchive2'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\x26hellip;'}, 'displayModeFull'));
</script>
</body>
</html>