Files
nexus/sreweekly/articles/303/06-day-20-to-deploy-or-not-to-deploy-that-is-the-question.html
2026-09-12 17:23:01 +08:00

1571 lines
59 KiB
HTML

<!DOCTYPE html>
<html dir='ltr' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'>
<head>
<link href='https://www.blogger.com/static/v1/widgets/4165133002-widget_css_bundle.css' rel='stylesheet' type='text/css'/>
<meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/>
<meta content='blogger' name='generator'/>
<link href='https://sysadvent.blogspot.com/favicon.ico' rel='icon' type='image/x-icon'/>
<link href='https://sysadvent.blogspot.com/2021/12/day-20-to-deploy-or-not-to-deploy-that.html' rel='canonical'/>
<link rel="alternate" type="application/atom+xml" title="sysadvent - Atom" href="https://sysadvent.blogspot.com/feeds/posts/default" />
<link rel="alternate" type="application/rss+xml" title="sysadvent - RSS" href="https://sysadvent.blogspot.com/feeds/posts/default?alt=rss" />
<link rel="service.post" type="application/atom+xml" title="sysadvent - Atom" href="https://www.blogger.com/feeds/3615332969083650973/posts/default" />
<link rel="alternate" type="application/atom+xml" title="sysadvent - Atom" href="https://sysadvent.blogspot.com/feeds/3478019044132222807/comments/default" />
<!--Can't find substitution for tag [blog.ieCssRetrofitLinks]-->
<meta content='https://sysadvent.blogspot.com/2021/12/day-20-to-deploy-or-not-to-deploy-that.html' property='og:url'/>
<meta content='Day 20 - To Deploy or Not to Deploy? That is the question.' property='og:title'/>
<meta content=' By: Jessica DeVita ( @ubergeekgirl ) Edited by: Jennifer Davis ( @sigje ) Deployment Decision-Making during the holidays amid...' property='og:description'/>
<title>
sysadvent: Day 20 - To Deploy or Not to Deploy? That is the question.
</title>
<style type='text/css'>@font-face{font-family:'Allerta';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/allerta/v19/TwMO-IAHRlkbx940YnYXSCiN9uc.woff2)format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDRrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcBBrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDBrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+1F00-1FFF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcAxrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0370-0377,U+037A-037F,U+0384-038A,U+038C,U+038E-03A1,U+03A3-03FF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcAhrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0307-0308,U+0590-05FF,U+200C-2010,U+20AA,U+25CC,U+FB1D-FB4F;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDxrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDhrBdwcoaaQwpBQ.woff2)format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcABrBdwcoaaQw.woff2)format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;}</style>
<style id='page-skin-1' type='text/css'><!--
/*
-----------------------------------------------
Blogger Template Style
Name: Minima
Designer: Douglas Bowman
URL: www.stopdesign.com
Date: 26 Feb 2004
Updated by: Blogger Team
----------------------------------------------- */
/* Variable definitions
====================
<Variable name="bgcolor" description="Page Background Color"
type="color" default="#fff">
<Variable name="textcolor" description="Text Color"
type="color" default="#333">
<Variable name="linkcolor" description="Link Color"
type="color" default="#58a">
<Variable name="pagetitlecolor" description="Blog Title Color"
type="color" default="#666">
<Variable name="descriptioncolor" description="Blog Description Color"
type="color" default="#999">
<Variable name="titlecolor" description="Post Title Color"
type="color" default="#c60">
<Variable name="bordercolor" description="Border Color"
type="color" default="#ccc">
<Variable name="sidebarcolor" description="Sidebar Title Color"
type="color" default="#999">
<Variable name="sidebartextcolor" description="Sidebar Text Color"
type="color" default="#666">
<Variable name="visitedlinkcolor" description="Visited Link Color"
type="color" default="#999">
<Variable name="bodyfont" description="Text Font"
type="font" default="normal normal 100% Georgia, Serif">
<Variable name="headerfont" description="Sidebar Title Font"
type="font"
default="normal normal 78% 'Trebuchet MS',Trebuchet,Arial,Verdana,Sans-serif">
<Variable name="pagetitlefont" description="Blog Title Font"
type="font"
default="normal normal 200% Georgia, Serif">
<Variable name="descriptionfont" description="Blog Description Font"
type="font"
default="normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif">
<Variable name="postfooterfont" description="Post Footer Font"
type="font"
default="normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif">
<Variable name="startSide" description="Side where text starts in blog language"
type="automatic" default="left">
<Variable name="endSide" description="Side where text ends in blog language"
type="automatic" default="right">
*/
/* Use this with templates/template-twocol.html */
body {
background:#ffffff;
margin:0;
color:#333333;
font:x-small Georgia Serif;
font-size/* */:/**/small;
font-size: /**/small;
text-align: center;
}
a:link {
color:#5588aa;
text-decoration:none;
}
a:visited {
color:#999999;
text-decoration:none;
}
a:hover {
color:#cc6600;
text-decoration:underline;
}
a img {
border-width:0;
}
/* Header
-----------------------------------------------
*/
#header-wrapper {
width:760px;
margin:0 auto 10px;
/*border:1px solid #cccccc;*/
}
#header-inner {
background-position: center;
margin-left: auto;
margin-right: auto;
}
#header {
margin: 5px;
/*border: 1px solid #cccccc;*/
text-align: center;
color:#666666;
}
#header h1 {
margin:5px 5px 0;
padding:15px 20px .25em;
line-height:1.2em;
text-transform:uppercase;
letter-spacing:.2em;
font: normal normal 200% Georgia, Serif;
}
#header a {
color:#666666;
text-decoration:none;
}
#header a:hover {
color:#666666;
}
#header .description {
margin:0 5px 5px;
padding:0 20px 15px;
max-width:800px;
text-transform:uppercase;
letter-spacing:.2em;
line-height: 1.4em;
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
color: #999999;
}
#header img {
margin-left: auto;
margin-right: auto;
}
/* Outer-Wrapper
----------------------------------------------- */
#outer-wrapper {
width: 760px;
margin:0 auto;
padding:10px;
text-align:left;
font: normal normal 100% Arimo;
}
#main-wrapper {
width: 510px;
float: left;
word-wrap: break-word; /* fix for long text breaking sidebar float in IE */
overflow: hidden; /* fix for long non-text content breaking IE sidebar float */
}
#sidebar-wrapper {
width: 220px;
float: right;
word-wrap: break-word; /* fix for long text breaking sidebar float in IE */
overflow: hidden; /* fix for long non-text content breaking IE sidebar float */
}
/* Headings
----------------------------------------------- */
h2 {
margin:1.5em 0 .75em;
font:normal normal 130% Allerta;
line-height: 1.4em;
text-transform:uppercase;
letter-spacing:.2em;
color:#999999;
}
/* Posts
-----------------------------------------------
*/
h2.date-header {
margin:1.5em 0 .5em;
}
.post {
margin:.5em 0 1.5em;
border-bottom:1px dotted #cccccc;
padding-bottom:1.5em;
}
.post h3 {
margin:.25em 0 0;
padding:0 0 4px;
font-size:140%;
font-weight:normal;
line-height:1.4em;
color:#cc6600;
}
.post h3 a, .post h3 a:visited, .post h3 strong {
display:block;
text-decoration:none;
color:#cc6600;
font-weight:normal;
}
.post h3 strong, .post h3 a:hover {
color:#333333;
}
.post-body {
margin:0 0 .75em;
line-height:1.6em;
}
.post-body blockquote {
line-height:1.3em;
}
.post-footer {
margin: .75em 0;
color:#999999;
text-transform:uppercase;
letter-spacing:.1em;
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
line-height: 1.4em;
}
.comment-link {
margin-left:.6em;
}
.post img {
padding:4px;
border:1px solid #cccccc;
}
.post blockquote {
margin:1em 20px;
}
.post blockquote p {
margin:.75em 0;
}
.post pre {
overflow: auto;
padding-left: 5px;
padding-right: 5px;
border: 1px solid #CCCCFF !important;
}
.post pre a {
text-decoration: underline;
}
/* Comments
----------------------------------------------- */
#comments h4 {
margin:1em 0;
font-weight: bold;
line-height: 1.4em;
text-transform:uppercase;
letter-spacing:.2em;
color: #999999;
}
#comments-block {
margin:1em 0 1.5em;
line-height:1.6em;
}
#comments-block .comment-author {
margin:.5em 0;
}
#comments-block .comment-body {
margin:.25em 0 0;
}
#comments-block .comment-footer {
margin:-.25em 0 2em;
line-height: 1.4em;
text-transform:uppercase;
letter-spacing:.1em;
}
#comments-block .comment-body p {
margin:0 0 .75em;
}
.deleted-comment {
font-style:italic;
color:gray;
}
#blog-pager-newer-link {
float: left;
}
#blog-pager-older-link {
float: right;
}
#blog-pager {
text-align: center;
}
.feed-links {
clear: both;
line-height: 2.5em;
}
/* Sidebar Content
----------------------------------------------- */
.sidebar {
color: #666666;
line-height: 1.5em;
}
.sidebar ul {
list-style:none;
margin:0 0 0;
padding:0 0 0;
}
.sidebar li {
margin:0;
padding-top:0;
padding-right:0;
padding-bottom:.25em;
padding-left:15px;
text-indent:-15px;
line-height:1.5em;
}
.sidebar .widget, .main .widget {
border-bottom:1px dotted #cccccc;
margin:0 0 1.5em;
padding:0 0 1.5em;
}
.main .Blog {
border-bottom-width: 0;
}
/* Profile
----------------------------------------------- */
.profile-img {
float: left;
margin-top: 0;
margin-right: 5px;
margin-bottom: 5px;
margin-left: 0;
padding: 4px;
border: 1px solid #cccccc;
}
.profile-data {
margin:0;
text-transform:uppercase;
letter-spacing:.1em;
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
color: #999999;
font-weight: bold;
line-height: 1.6em;
}
.profile-datablock {
margin:.5em 0 .5em;
}
.profile-textblock {
margin: 0.5em 0;
line-height: 1.6em;
}
.profile-link {
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
text-transform: uppercase;
letter-spacing: .1em;
}
/* Footer
----------------------------------------------- */
#footer {
width:760px;
clear:both;
margin:0 auto;
padding-top:15px;
line-height: 1.6em;
text-transform:uppercase;
letter-spacing:.1em;
text-align: center;
}
@media only screen and (max-width:480px) {
a:link {
color: #d12a39;
}
.mobile-link-button {
background-color: #d12a39;
}
.mobile-link-button a:link, .mobile-link-button a:visited {
color: #ffffff;
}
}
--></style>
<link href='https://www.blogger.com/dyncss/3615332969083650973/authorization.css?zx=24cd04b2-38ff-45f4-a3f0-f0a07f75ec8f' media='none' onload='if(media!=&#39;all&#39;)media=&#39;all&#39;' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyncss/3615332969083650973/authorization.css?zx=24cd04b2-38ff-45f4-a3f0-f0a07f75ec8f' rel='stylesheet'/></noscript>
<meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/>
<meta name='google-adsense-platform-domain' content='blogspot.com'/>
<!-- data-ad-client=ca-pub-6194074709963145 -->
</head>
<body>
<div class='navbar section' id='navbar'><div class='widget Navbar' data-version='1' id='Navbar1'><script type="text/javascript">
function setAttributeOnload(object, attribute, val) {
if(window.addEventListener) {
window.addEventListener('load',
function(){ object[attribute] = val; }, false);
} else {
window.attachEvent('onload', function(){ object[attribute] = val; });
}
}
</script>
<div id="navbar-iframe-container"></div>
<script type="text/javascript" src="https://apis.google.com/js/platform.js"></script>
<script type="text/javascript">
gapi.load("gapi.iframes:gapi.iframes.style.bubble", function() {
if (gapi.iframes && gapi.iframes.getContext) {
gapi.iframes.getContext().openChild({
url: 'https://www.blogger.com/navbar/3615332969083650973?po\x3d3478019044132222807\x26origin\x3dhttps://sysadvent.blogspot.com',
where: document.getElementById("navbar-iframe-container"),
id: "navbar-iframe"
});
}
});
</script><script type="text/javascript">
(function() {
var script = document.createElement('script');
script.type = 'text/javascript';
script.src = '//pagead2.googlesyndication.com/pagead/js/google_top_exp.js';
var head = document.getElementsByTagName('head')[0];
if (head) {
head.appendChild(script);
}})();
</script>
</div></div>
<div id='outer-wrapper'>
<div id='wrap2'>
<!-- skip links for text browsers -->
<span id='skiplinks' style='display:none;'>
<a href='#main'>
skip to main
</a>
|
<a href='#sidebar'>
skip to sidebar
</a>
</span>
<div id='header-wrapper'>
<div class='header section' id='header'><div class='widget Header' data-version='1' id='Header1'>
<div id='header-inner'>
<a href='https://sysadvent.blogspot.com/' style='display: block'>
<img alt='sysadvent' height='240px; ' id='Header1_headerimg' src='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiYVlIMEU0sXvTBtcY5iFj5QtiT3Klf0S3sgSamkTPsjU1QGvOWHA-6yq173iHys3-ImrIFY26KIPuKEoSNSbitvtnXsZU6b8YPGQ48ir-8Jo2NFrTsaiO_Nl9W9XLt_wyWhTM_16_HeN0/s750/sys-advent-long.png' style='display: block' width='750px; '/>
</a>
</div>
</div></div>
</div>
<div id='content-wrapper'>
<div id='crosscol-wrapper' style='text-align:center'>
<div class='crosscol no-items section' id='crosscol'></div>
</div>
<div id='main-wrapper'>
<div class='main section' id='main'><div class='widget Blog' data-version='1' id='Blog1'>
<div class='blog-posts hfeed'>
<!--Can't find substitution for tag [defaultAdStart]-->
<div class="date-outer">
<h2 class='date-header'>
<span>
December 20, 2021
</span>
</h2>
<div class="date-posts">
<div class='post-outer'>
<div class='post hentry'>
<a name='3478019044132222807'></a>
<h3 class='post-title entry-title'>
<a href='https://sysadvent.blogspot.com/2021/12/day-20-to-deploy-or-not-to-deploy-that.html'>
Day 20 - To Deploy or Not to Deploy? That is the question.
</a>
</h3>
<div class='post-header-line-1'></div>
<div class='post-body entry-content'>
<p>
By: Jessica DeVita (<a href="https://twitter.com/ubergeekgirl">@ubergeekgirl</a>)<br />
Edited by: Jennifer Davis (<a href="https://twitter.com/sigje">@sigje</a>)
</p>
<p>
Deployment Decision-Making during the holidays amid the COVID19 Pandemic
</p>
<p>
<em>A sneak peek into my forthcoming MSc. thesis in Human Factors and Systems
Safety, Lund University</em>.
</p>
<p>
Web services that millions of us depend on for work and entertainment require
vast compute resources (servers, nodes, networking) and interdependent software
services, each configured in specialized ways. The experts who work on these
distributed systems are under <em>enormous</em> pressure to deploy new features,
<em>and</em> keep the services running, so deployment decisions are happening
hundreds or thousands of times every day. While automated testing and deployment
pipelines allow for frequent production changes, an engineer making a change
wants confidence that the automated testing system is working. However,
automating the testing pipeline makes the test-and-release process more opaque
to the engineer, making it difficult to troubleshoot.
</p>
<p>
</p>
<p>
When an incident occurs, the decisions preceding the event may be brought under
a microscope, often concluding that &#8220;human error&#8221; was the cause. As society
increasingly relies on web services, it is imperative to understand the
tradeoffs and considerations engineers face when they decide to deploy a change
into production. The themes uncovered through this research underscore the
complexity of engineering work in production environments and highlight the role
of relationships with co-workers and management on deployment decision-making.
</p>
<h2>There&#8217;s No Place Like Production</h2>
<p>
Many deployments are uneventful and proceed without issues, but unforeseen
permissions issues, network latency, sudden increases in demand, and security
vulnerabilities may only manifest in production.<em> </em>When asked to describe
a recent deployment decision, engineers reported intense feelings of uncertainty
as they could not predict how their change would interact with changes elsewhere
in the system. More automation isn&#8217;t always the solution, as one engineer
explains:
</p>
<p>
<em>&#8220;I can&#8217;t promise that when it goes out to the entire production fleet
that the timing won&#8217;t be wrong. It&#8217;s a giant Rube Goldberg of a race condition.
It feels like a technical answer to a human problem. I&#8217;ve seen people set up
Jenkins jobs with locks that prevent other jobs from running until it&#8217;s
complete. How often does it blow up in your face and fail to release the lock?
If a change is significant enough to worry about, there should be a human
shepherding it. Know each other&#8217;s names. Just talk to each other; it&#8217;s not that
hard.&#8221;</em>
</p>
<h2>Decision-making Under Pressure</h2>
<p>
<em>&#8220;The effects of an action can be totally different, if performed too
early or too late. But the right time is not clock time: it depends upon the
precise state of the process evolution&#8221;</em> (De Keyser, 1990).
</p>
<p>
Some engineers were under pressure to deploy fixes and features before the
holidays, while other engineers were constrained by a "<em>code freeze</em>",
when during certain times of the year, they &#8220;<em>can&#8217;t make significant
production changes that aren&#8217;t trivial or that fix something</em>&#8221;. One engineer
felt that they could continue to deploy to their test and staging environments
but warned, &#8220;... <em>a lot of things in a staging environment waiting to go out
can compound the risk of the deployments.&#8221;</em>
</p>
<p>
Responding to an incident or outage at any time of the year is challenging, but
even more so because of <em>&#8220;oddities that happen around holidays&#8221;</em> and
additional pressures from management, customers, and the engineers themselves.
Pairing or working together was often done as a means to increase confidence in
decision making. Pairing resulted in joint decisions, as engineers described
actions and decisions with &#8220;we&#8221;. <em>&#8220;So that was a late night. When I hit
something like that, it involves a lot more point-by-point communications with
my counterpart. For example,&#8221;I'm going to try this, do you agree this is a good
thing? What are we going to type in?&#8221;</em>
</p>
<p>
Engineers often grappled with "<em>clock time</em>" and reported that they made
certain sacrifices to &#8220;<em>buy more time</em>&#8221; to make further decisions. An
engineer expressed that a change <em>&#8220;couldn&#8217;t be decided under pressure in the
moment&#8221;</em> so they implemented a temporary measure.<em> </em>Fully aware of
the potential for their change to trigger new and different problems, engineers
wondered what they could do <em>&#8220;without making it worse&#8221;</em>.
</p>
<p>
When triaging unexpected complications, engineers sometimes <em>&#8220;went down
rabbit holes&#8221;</em>, exemplifying a cognitive fixation known as a &#8220;failure to
revise&#8221; (Woods & Cook, 1999). Additionally, having pertinent knowledge does not
guarantee that engineers can apply it in a given situation. For example, one
engineer recounted their experience during an incident on Christmas Eve:
</p>
<p>
<em>&#8220;...what happens to all of these volumes in the meantime? And so then
we're just thinking of the possible problems, and then [my co-worker] suggested
resizing it. And I said, &#8216;Oh, can you do that to a root volume?&#8217; &#8216;Cause I hadn't
done that before. I know you can do it to other volumes, but not the
root.&#8217;&#8221;</em>
</p>
<p>
Incidents were even <em>more</em> surprising in systems that rarely fail. For
one engineer working on a safety critical system, responding to an incident was
like a <em>&#8220;third level of panic&#8221;</em>.
</p>
<h3>Safety Practices</h3>
<p>
The ability to roll back a deployment was a critically important capability that
for one engineer was only possible because they had &#8220;<em>proper safety practices
in place&#8221;.</em> However, rollbacks were not guaranteed to work, as another
engineer explained:
</p>
<p>
<em>&#8220;It was a fairly catastrophic failure because the previous migration
with a typo had partially applied and not rolled back properly when it failed.
The update statement failed, but the migration tool didn&#8217;t record that it had
attempted the migration, because it had failed. It did not roll back the
addition, which I believed it would have done automatically&#8221;.</em>
</p>
<h3>Sleep Matters</h3>
<p>
One engineer described how they felt that being woken up several times during
the night was a direct cause of taking down production during their on-call
shift:
</p>
<p>
<em>&#8220;I didn't directly connect that what I had done to try to fix the page
was what had caused the outage because of a specific symptom I was seeing&#8230; I
think if I had more sleep it would have gotten fixed sooner&#8221;. </em>
</p>
<p>
Despite needing <em>&#8220;moral support&#8221;</em>, engineers didn&#8217;t want to wake up their
co-workers in different time zones: <em>&#8220;You don't just have the stress of the
company on your shoulders. You've got the stress of paying attention to what
you're doing and the stress of having to do this late at night.&#8221;</em> This was
echoed in another engineer&#8217;s reluctance to page co-workers at night as they
&#8220;<em>thought they could try one more thing, but it&#8217;s hard to be self-aware in
the middle of the night when things are broken, we&#8217;re stressed and tired&#8221;.</em>
</p>
<p>
Engineers also talked about the impacts of a lack of sleep on their
effectiveness at work as <em>&#8220;not operating on all cylinders&#8221;</em>, and no
different than having 3 or 4 drinks: &#8220;<em>It could happen in the middle of the
night when you're already tired and a little delirious. It's a form of
intoxication in my book.</em>&#8221;
</p>
<h3>Blame Culture</h3>
<p>
<em>&#8220;What's the mean time to innocence? How quickly can you show that it's not a
problem with your system?&#8221;</em>
</p>
<p>
Some engineers described feeling that management was blameful after incidents
and untruthful about priorities. For example, an engineer described the
aftermath of a difficult database migration: <em>&#8220;Upper management was not
straightforward with us. We compromised our technical integrity and our
standards for ourselves because we were told we had to&#8221;.</em>
</p>
<p>
Another engineer described a blameful culture during post-incident review
meetings:
</p>
<p>
<em>&#8220;It is a very nerve-wracking and fraught experience to be asked to come
to a meeting with the directors and explain what happened and why your product
broke. And because this is an interwoven system, everybody's dependent on us and
if something happens, then it&#8217;s like &#8216;you need to explain what happened because
it hurt us.&#8221;</em>
</p>
<p>
Engineers described their errors as <em>"honest mistakes'' </em>as they made
sense of these events after the fact. Some felt a strong sense of personal
failure, and that their actions were the cause of the incident, as this engineer
describes:
</p>
<p>
<em>&#8220;We are supposed to follow a blameless process, but a lot of the time
people self-blame. You can't really shut it down that much because frankly they
are very causal events. I'm not the only one who can't really let go of it. I
know it was because of what I did.&#8221;</em>
</p>
<p>
Not all engineers felt they could take <em>&#8220;interpersonal risks&#8221;</em> or admit a
lack of knowledge without fear of <em>&#8220;being seen as incompetent&#8221;</em>.
Synthesizing theories of psychological safety with this study&#8217;s findings, it
seems clear that environments of psychological safety may increase engineers&#8217;
confidence in decision making (Edmondson, 2002).
</p>
<h2>What Would They Change?</h2>
<p>
Engineers were asked &#8220;If you could wave a magic wand, what would you change
about your current environment that would help you feel more confident or safe
in your day-to-day deployment decisions?
</p>
<p>
In addition to <em>&#8220;faster CI and pre-deployments&#8221;, </em>engineers overarchingly
spoke about needing better testing. One participant wanted a better way to test
front-end code end-to-end, <em>"I return to this space every few years and am a
bit surprised that this still is so hard to get right&#8221;. </em>In another mention
of improved testing, an engineer wanted &#8220;<em>integration tests that exercise the
subject component along with the graph of dependencies (other components,
databases, etc.), using only public APIs. I.e., no "direct to database"
fixtures, no mocking&#8221;.</em>
</p>
<h2>Wrapping Up</h2>
<p>
Everything about engineers&#8217; work was made more difficult in the face of a global
pandemic. In the &#8220;before times&#8221; engineers could <em>"swivel their chair</em>&#8221; to
get a <em>"second set of eyes" </em>on from co-workers before deploying. While
some engineers in the study had sophisticated deployment automation, others
spoke of manual workarounds with heroic scripts written &#8216;on the fly&#8217; to repair
the system when it failed. Engineers grappled with the complexities of
automation, and the risk and uncertainty associated with decisions to deploy.
Most engineers using tools to automate and manage configurations did
<em>not</em> experience relief in their workload. They had to maintain skills in
manual interventions when the automation did not work as expected or when they
could not discern the machine&#8217;s state. Such experiences highlight the continued
relevance of Lisanne Bainbridge&#8217;s (1983) research on the Ironies of Automation
which found that &#8220;the more advanced a control system is, the more crucial the
role of the operator&#8221;.
</p>
<p>
This study revealed that deployment decisions <em>cannot</em> be understood
independently from the social systems, rituals, and organizational structures in
which they occurred (Pettersen, McDonald, & Engen, 2010). So when a deployment
decision results in an incident or outage, instead of blaming the engineer,
consider the words of James Reason (1990) who said &#8220;<em>...operators tend to be
the inheritors of system defects&#8230;adding the final garnish to a lethal brew whose
ingredients have already been long in the cooking&#8221;.</em> Engineers may bring
their previous experiences to deployment decisions, but the tools and conditions
of their work environment, historical events, power structures, and hierarchy
are what <em>&#8220;enables and sets the stage for all human action.&#8221;</em> (Dekker &
Nyce, 2014, p. 47).
</p>
<p>
____
</p>
<p>
This is an excerpt from Jessica&#8217;s forthcoming thesis. If you&#8217;re interested in
learning more about this deployment decision-making study or would like to
explore future research opportunities, send Jessica a message on <a href="twitter.com/ubergeekgirl">Twitter</a>.
</p>
<p>
<strong>References</strong>
</p>
<p>
Bainbridge, L. (1983). IRONIES OF AUTOMATION. In G. Johannsen &amp; J. E.
Rijnsdorp (Eds.), <em>Analysis, Design and Evaluation of Man&#8211;Machine
Systems</em> (pp. 129&#8211;135). Pergamon.
</p>
<p>
De Keyser, V. (1990). Temporal decision making in complex environments.
<em>Philosophical Transactions of the Royal Society of London. Series B,
Biological Sciences</em>, 327(1241), 569&#8211;576.
</p>
<p>
Dekker, S. W. A., &amp; Nyce, J. M. (2014). There is safety in power, or power
in safety. <em>Safety Science</em>, 67, 44&#8211;49.
</p>
<p>
Edmondson, A. C. (2002). <em>Managing the risk of learning: Psychological
safety in work teams</em>. Citeseer.
</p>
<p>
Pettersen, K. A., McDonald, N., &amp; Engen, O. A. (2010). Rethinking the role
of social theory in socio-technical analysis: a critical realist approach to
aircraft maintenance. <em>Cognition, Technology &amp; Work</em>, 12(3), 181&#8211;191.
</p>
<p>
Reason, J. (1990). <em>Human Error</em> (pp. 173&#8211;216). Cambridge University
Press.
</p>
<p>
Woods, D. D., &amp; Cook, R. I. (1999). Perspectives on Human Error: Hindsight
Bias and Local Rationality. In <em>In F. Durso (Ed.) Handbook of Applied
Cognitive Psychology</em>. Retrieved 9 June 2021 from
http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.474.3161
</p>
<div style='clear: both;'></div>
</div>
<div class='post-footer'>
<div class='post-footer-line post-footer-line-1'>
<span class='post-author vcard'>
Posted by
<span class='fn'>
sigje
</span>
</span>
<span class='post-timestamp'>
</span>
<span class='post-comment-link'>
</span>
<span class='post-icons'>
<span class='item-action'>
<a href='https://www.blogger.com/email-post/3615332969083650973/3478019044132222807' title='Email Post'>
<img alt="" class="icon-action" height="13" src="//www.blogger.com/img/icon18_email.gif" width="18">
</a>
</span>
<span class='item-control blog-admin pid-1271197227'>
<a href='https://www.blogger.com/post-edit.g?blogID=3615332969083650973&postID=3478019044132222807&from=pencil' title='Edit Post'>
<img alt="" class="icon-action" height="18" src="//img2.blogblog.com/img/icon18_edit_allbkg.gif" width="18">
</a>
</span>
</span>
</div>
<div class='post-footer-line post-footer-line-2'>
<span class='post-labels'>
Labels:
<a href='https://sysadvent.blogspot.com/search/label/decision-making' rel='tag'>
decision-making
</a>
,
<a href='https://sysadvent.blogspot.com/search/label/deployment' rel='tag'>
deployment
</a>
,
<a href='https://sysadvent.blogspot.com/search/label/Human%20Factors' rel='tag'>
Human Factors
</a>
,
<a href='https://sysadvent.blogspot.com/search/label/research' rel='tag'>
research
</a>
</span>
</div>
<div class='post-footer-line post-footer-line-3'></div>
</div>
</div>
<div class='comments' id='comments'>
<a name='comments'></a>
<h4>
No comments
:
</h4>
<div id='Blog1_comments-block-wrapper'>
<dl class='' id='comments-block'>
</dl>
</div>
<p class='comment-footer'>
<a href='https://www.blogger.com/comment/fullpage/post/3615332969083650973/3478019044132222807' onclick=''>
Post a Comment
</a>
</p>
<div id='backlinks-container'>
<div id='Blog1_backlinks-container'>
</div>
</div>
</div>
</div>
</div></div>
<!--Can't find substitution for tag [adEnd]-->
</div>
<div class='blog-pager' id='blog-pager'>
<span id='blog-pager-newer-link'>
<a class='blog-pager-newer-link' href='https://sysadvent.blogspot.com/2021/12/day-22-so-youre-incident-commander-now.html' id='Blog1_blog-pager-newer-link' title='Newer Post'>
Newer Post
</a>
</span>
<span id='blog-pager-older-link'>
<a class='blog-pager-older-link' href='https://sysadvent.blogspot.com/2021/12/day-19-into-world-of-chaos-engineering.html' id='Blog1_blog-pager-older-link' title='Older Post'>
Older Post
</a>
</span>
<a class='home-link' href='https://sysadvent.blogspot.com/'>
Home
</a>
</div>
<div class='clear'></div>
<div class='post-feeds'>
<div class='feed-links'>
Subscribe to:
<a class='feed-link' href='https://sysadvent.blogspot.com/feeds/3478019044132222807/comments/default' target='_blank' type='application/atom+xml'>
Post Comments
(
Atom
)
</a>
</div>
</div>
</div></div>
</div>
<div id='sidebar-wrapper'>
<div class='sidebar section' id='sidebar'><div class='widget HTML' data-version='1' id='HTML2'>
<h2 class='title'>
What is sysadvent?
</h2>
<div class='widget-content'>
<p>
One article for each day of December, ending on the 25th article.
</p><p>
With the goals of sharing, openness, and mentoring, we aim to provide great articles about systems administration topics written by fellow sysadmins.
</p><p>
Want to get involved? Join the <a href="http://groups.google.com/group/sysadvent">mailing list</a>!
<p></p></p>
</div>
<div class='clear'></div>
</div><div class='widget Subscribe' data-version='1' id='Subscribe1'>
<div style='white-space:nowrap'>
<h2 class='title'>
Subscribe
</h2>
<div class='widget-content'>
<div class='subscribe-wrapper subscribe-type-POST'>
<div class='subscribe expanded subscribe-type-POST' id='SW_READER_LIST_Subscribe1POST' style='display:none;'>
<div class='top'>
<span class='inner' onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Posts
</span>
<div class='feed-reader-links'>
<a class='feed-reader-link' href='http://www.netvibes.com/subscribe.php?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-netvibes.png'/>
</a>
<a class='feed-reader-link' href='http://add.my.yahoo.com/content?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-yahoo.png'/>
</a>
<a class='feed-reader-link' href='https://sysadvent.blogspot.com/feeds/posts/default' target='_blank'>
<img align='absmiddle' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Atom
</a>
</div>
</div>
<div class='bottom'></div>
</div>
<div class='subscribe' id='SW_READER_LIST_CLOSED_Subscribe1POST' onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
<div class='top'>
<span class='inner'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<span onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Posts
</span>
</span>
</div>
<div class='bottom'></div>
</div>
</div>
<div class='subscribe-wrapper subscribe-type-PER_POST'>
<div class='subscribe expanded subscribe-type-PER_POST' id='SW_READER_LIST_Subscribe1PER_POST' style='display:none;'>
<div class='top'>
<span class='inner' onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Comments
</span>
<div class='feed-reader-links'>
<a class='feed-reader-link' href='http://www.netvibes.com/subscribe.php?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2F3478019044132222807%2Fcomments%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-netvibes.png'/>
</a>
<a class='feed-reader-link' href='http://add.my.yahoo.com/content?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2F3478019044132222807%2Fcomments%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-yahoo.png'/>
</a>
<a class='feed-reader-link' href='https://sysadvent.blogspot.com/feeds/3478019044132222807/comments/default' target='_blank'>
<img align='absmiddle' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Atom
</a>
</div>
</div>
<div class='bottom'></div>
</div>
<div class='subscribe' id='SW_READER_LIST_CLOSED_Subscribe1PER_POST' onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
<div class='top'>
<span class='inner'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<span onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Comments
</span>
</span>
</div>
<div class='bottom'></div>
</div>
</div>
<div style='clear:both'></div>
</div>
</div>
<div class='clear'></div>
</div><div class='widget BlogArchive' data-version='1' id='BlogArchive2'>
<h2>
Blog Archive
</h2>
<div class='widget-content'>
<div id='ArchiveList'>
<div id='BlogArchive2_ArchiveList'>
<ul class='hierarchy'>
<li class='archivedate expanded'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy toggle-open'>
&#9660;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2021/'>
2021
</a>
<span class='post-count' dir='ltr'>
(
22
)
</span>
<ul class='hierarchy'>
<li class='archivedate expanded'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy toggle-open'>
&#9660;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2021/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
22
)
</span>
<ul class='posts'>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-23-what-is-ebpf.html'>
Day 23 - What is eBPF?
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-22-so-youre-incident-commander-now.html'>
Day 22 - So, You&#39;re Incident Commander, Now What?
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-20-to-deploy-or-not-to-deploy-that.html'>
Day 20 - To Deploy or Not to Deploy? That is the q...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-19-into-world-of-chaos-engineering.html'>
Day 19 - Into the World of Chaos Engineering
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-18-minimizing-false-positive.html'>
Day 18 - Minimizing False Positive Monitoring Aler...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-17-death-to-localhost-benefits-of.html'>
Day 17 - Death to Localhost: The Benefits of Devel...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-16-setting-up-k3s-in-your-home-lab.html'>
Day 16 - Setting up k3s in your home lab
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-15-introduction-to-pagerduty-api.html'>
Day 15 - Introduction to the PagerDuty API
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-14-whats-in-job-description-and-who.html'>
Day 14 - What&#39;s in a job description (and who does...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-13-ephemeral-pr-environments.html'>
Day 13 - Ephemeral PR Environments: Enabling autom...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-12-terraform-refactoring.html'>
Day 12 - Terraform Refactoring
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-11-moving-from-engineering-manager.html'>
Day 11 - Moving from Engineering Manager to IC
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-10-assembling-your-year-in-review.html'>
Day 10 - Assembling Your Year In Review
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/3-things-parenting-taught-me-about.html'>
Day 9 - 3 things parenting taught me about system ...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-8-d-for-sres.html'>
Day 8 - D&amp;D for SREs
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-7-baking-multi-architecture-docker.html'>
Day 7 - Baking Multi-architecture Docker Images
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/day-6-more-to-come-tomorrow.html'>
Day 6 - More to come tomorrow!
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/least-privilege-using-strace.html'>
Day 5 - Least Privilege using strace
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/gwlb-panacea-for-cloud-dmz-on-aws.html'>
Day 4 - GWLB: Panacea for Cloud DMZ on AWS
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/keeping-config-management-simple-with.html'>
Day 3 - Keeping Config Management Simple with Itamae
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/reliability-as-product-feature.html'>
Day 2 - Reliability as a Product Feature
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2021/12/the-myths-and-magic-in-my-search-for.html'>
Day 1 - The Myths and the Magic in My Search for A...
</a>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2019/'>
2019
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2019/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2017/'>
2017
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2017/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2016/'>
2016
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2016/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2015/'>
2015
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2015/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2014/'>
2014
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2014/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2013/'>
2013
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2013/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2012/'>
2012
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2012/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2011/'>
2011
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2011/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2010/'>
2010
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2010/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2009/'>
2009
</a>
<span class='post-count' dir='ltr'>
(
26
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2009/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
26
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2008/'>
2008
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2008/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
</div>
</div>
<div class='clear'></div>
</div>
</div></div>
</div>
<!-- spacer for skins that want sidebar and main to be the same height-->
<div class='clear'>
&#160;
</div>
</div>
<!-- end content-wrapper -->
<div id='footer-wrapper'>
<div class='footer no-items section' id='footer'></div>
</div>
</div>
</div>
<!-- end outer-wrapper -->
<script type='text/javascript'>
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
</script>
<script type='text/javascript'>
try {
var pageTracker = _gat._getTracker("UA-6522917-1");
pageTracker._trackPageview();
} catch(err) {}</script>
<script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/1363620594-widgets.js"></script>
<script type='text/javascript'>
window['__wavt'] = 'AAPvtVnWWopRWpAQwbrbGpbOF9bY:1789189994084';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d3615332969083650973','//sysadvent.blogspot.com/2021/12/day-20-to-deploy-or-not-to-deploy-that.html','3615332969083650973');
_WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '3615332969083650973', 'title': 'sysadvent', 'url': 'https://sysadvent.blogspot.com/2021/12/day-20-to-deploy-or-not-to-deploy-that.html', 'canonicalUrl': 'https://sysadvent.blogspot.com/2021/12/day-20-to-deploy-or-not-to-deploy-that.html', 'homepageUrl': 'https://sysadvent.blogspot.com/', 'searchUrl': 'https://sysadvent.blogspot.com/search', 'canonicalHomepageUrl': 'https://sysadvent.blogspot.com/', 'blogspotFaviconUrl': 'https://sysadvent.blogspot.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': false, 'httpsEnabled': true, 'enabledCommentProfileImages': false, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': '', 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22sysadvent - RSS\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/3615332969083650973/posts/default\x22 /\x3e\n\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/3478019044132222807/comments/default\x22 /\x3e\n', 'meTag': '', 'adsenseClientId': 'ca-pub-6194074709963145', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'isGoogleEverywhereLinkTooltipEnabled': true, 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/1890a9484e710b1c', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'item', 'postId': '3478019044132222807', 'pageName': 'Day 20 - To Deploy or Not to Deploy? That is the question.', 'pageTitle': 'sysadvent: Day 20 - To Deploy or Not to Deploy? That is the question.'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Day 20 - To Deploy or Not to Deploy? That is the question.', 'description': ' By: Jessica DeVita ( @ubergeekgirl ) Edited by: Jennifer Davis ( @sigje ) Deployment Decision-Making during the holidays amid...', 'url': 'https://sysadvent.blogspot.com/2021/12/day-20-to-deploy-or-not-to-deploy-that.html', 'type': 'item', 'isSingleItem': true, 'isMultipleItems': false, 'isError': false, 'isPage': false, 'isPost': true, 'isHomepage': false, 'isArchive': false, 'isLabelSearch': false, 'postId': 3478019044132222807}}]);
_WidgetManager._RegisterWidget('_NavbarView', new _WidgetInfo('Navbar1', 'navbar', document.getElementById('Navbar1'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/700075392-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/828616780-lightbox_bundle.css'}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML2', 'sidebar', document.getElementById('HTML2'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_SubscribeView', new _WidgetInfo('Subscribe1', 'sidebar', document.getElementById('Subscribe1'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive2', 'sidebar', document.getElementById('BlogArchive2'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\x26hellip;'}, 'displayModeFull'));
</script>
</body>
</html>