Files
nexus/sreweekly/articles/182/01-continuous-verification-of-friday-deploys.html
2026-09-12 17:23:01 +08:00

382 lines
40 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="profile" href="http://gmpg.org/xfn/11">
<link rel="pingback" href="https://willgallego.com/xmlrpc.php">
<title>Continuous Verification of Friday Deploys &#8211; Will Gallego</title>
<meta name='robots' content='max-image-preview:large' />
<style>img:is([sizes="auto" i], [sizes^="auto," i]) { contain-intrinsic-size: 3000px 1500px }</style>
<link rel='dns-prefetch' href='//fonts.googleapis.com' />
<link rel="alternate" type="application/rss+xml" title="Will Gallego &raquo; Feed" href="https://willgallego.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="Will Gallego &raquo; Comments Feed" href="https://willgallego.com/comments/feed/" />
<link rel="alternate" type="application/rss+xml" title="Will Gallego &raquo; Continuous Verification of Friday Deploys Comments Feed" href="https://willgallego.com/2019/08/23/continuous-verification-of-friday-deploys/feed/" />
<script type="text/javascript">
/* <![CDATA[ */
window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/16.0.1\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/16.0.1\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/willgallego.com\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.8.8"}};
/*! This file is auto-generated */
!function(s,n){var o,i,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),a=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===a[t]})}function u(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);for(var n=e.getImageData(16,16,1,1),a=0;a<n.data.length;a++)if(0!==n.data[a])return!1;return!0}function f(e,t,n,a){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!a(e,"\ud83e\udedf")}return!1}function g(e,t,n,a){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):s.createElement("canvas"),o=r.getContext("2d",{willReadFrequently:!0}),i=(o.textBaseline="top",o.font="600 32px Arial",{});return e.forEach(function(e){i[e]=t(o,e,n,a)}),i}function t(e){var t=s.createElement("script");t.src=e,t.defer=!0,s.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",i=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){s.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+g.toString()+"("+[JSON.stringify(i),f.toString(),p.toString(),u.toString()].join(",")+"));",a=new Blob([e],{type:"text/javascript"}),r=new Worker(URL.createObjectURL(a),{name:"wpTestEmojiSupports"});return void(r.onmessage=function(e){c(n=e.data),r.terminate(),t(n)})}catch(e){}c(n=g(i,f,p,u))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings);
/* ]]> */
</script>
<style id='wp-emoji-styles-inline-css' type='text/css'>
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
</style>
<link rel='stylesheet' id='wp-block-library-css' href='https://willgallego.com/wp-includes/css/dist/block-library/style.min.css?ver=6.8.8' type='text/css' media='all' />
<style id='classic-theme-styles-inline-css' type='text/css'>
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
</style>
<style id='global-styles-inline-css' type='text/css'>
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}
:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}
:root :where(.wp-block-pullquote){font-size: 1.5em;line-height: 1.6;}
</style>
<link rel='stylesheet' id='arcade-basic-stylesheet-css' href='https://willgallego.com/wp-content/themes/arcade-basic/style.css?ver=6.8.8' type='text/css' media='all' />
<link rel='stylesheet' id='arcade-basic-fonts-css' href='//fonts.googleapis.com/css?family=Megrim|Raleway|Open+Sans:400,400italic,700,700italic' type='text/css' media='all' />
<link rel='stylesheet' id='font-awesome-css' href='https://willgallego.com/wp-content/themes/arcade-basic/library/css/font-awesome.css?ver=4.7.0' type='text/css' media='all' />
<script type="text/javascript" src="https://willgallego.com/wp-includes/js/jquery/jquery.min.js?ver=3.7.1" id="jquery-core-js"></script>
<script type="text/javascript" src="https://willgallego.com/wp-includes/js/jquery/jquery-migrate.min.js?ver=3.4.1" id="jquery-migrate-js"></script>
<link rel="https://api.w.org/" href="https://willgallego.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://willgallego.com/wp-json/wp/v2/posts/449" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://willgallego.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress 6.8.8" />
<link rel="canonical" href="https://willgallego.com/2019/08/23/continuous-verification-of-friday-deploys/" />
<link rel='shortlink' href='https://willgallego.com/?p=449' />
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://willgallego.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fwillgallego.com%2F2019%2F08%2F23%2Fcontinuous-verification-of-friday-deploys%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://willgallego.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fwillgallego.com%2F2019%2F08%2F23%2Fcontinuous-verification-of-friday-deploys%2F&#038;format=xml" />
<style>
.container { max-width: 1170px; }
</style>
<style type="text/css">.recentcomments a{display:inline !important;padding:0 !important;margin:0 !important;}</style></head>
<body class="wp-singular post-template-default single single-post postid-449 single-format-standard wp-theme-arcade-basic no-fittext basic">
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-106878020-1"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'UA-106878020-1');
</script>
<div id="page">
<header id="header">
<nav id="site-navigation" class="navbar navbar-inverse navbar-fixed-top" role="navigation">
<h3 class="sr-only">Main menu</h3>
<a class="sr-only" href="#primary" title="Skip to content">Skip to content</a>
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<div class="collapse navbar-collapse">
<ul id="menu-links" class="nav navbar-nav"><li id="menu-item-5" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-home menu-item-5"><a href="http://willgallego.com">Home</a></li>
<li id="menu-item-36" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-36"><a href="https://willgallego.com/about/">About</a></li>
<li id="menu-item-207" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-207"><a href="https://willgallego.com/talks/">Talks</a></li>
<li id="menu-item-7" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-7"><a target="_blank" href="https://twitter.com/wcgallego">Twitter</a></li>
<li id="menu-item-8" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-8"><a target="_blank" href="https://github.com/wcgallego">Github</a></li>
<li id="menu-item-651" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-651"><a target="_blank" rel="me" href="https://mastodon.social/@gallego">Mastodon</a></li>
</ul> </div>
</nav><!-- #site-navigation -->
<div class="title-card-wrapper">
<div class="title-card">
<div id="site-meta">
<div id="site-title">
<a href="https://willgallego.com" title="Will Gallego" rel="home">Will Gallego</a>
</div>
<i class="fa fa-gamepad"></i>
<div id="site-description">
Software Engineering, Blame Awareness, and gif with a soft ‘G’ </div>
<a href="#" id="more-site" class="btn btn-default btn-lg">See More</a>
</div>
<img class="header-img" src="https://willgallego.com/wp-content/uploads/2017/09/cropped-Trinity_College_Long_Room_ceiling.jpg" alt="" />
</div>
</div>
</header>
<main>
<div class="container">
<div class="row">
<div id="primary" class="col-md-8 hfeed">
<article id="post-449" class="clearfix post-449 post type-post status-publish format-standard has-post-thumbnail hentry category-human-factors category-tech tag-deploys tag-resilience-engineering xfolkentry">
<h1 class="entry-title">Continuous Verification of Friday Deploys</h1>
<div class="entry-meta">
by <span class="vcard author"><span class="fn"><a href="https://willgallego.com/author/will/" title="Posts by Will Gallego" rel="author">Will Gallego</a></span></span>&nbsp;on&nbsp;<a href="https://willgallego.com/2019/08/23/continuous-verification-of-friday-deploys/" class="time"><time class="date published updated" datetime="2019-08-23">August 23, 2019</time></a>&nbsp;in&nbsp;<a href="https://willgallego.com/category/human-factors/" rel="category tag">Human Factors</a>, <a href="https://willgallego.com/category/tech/" rel="category tag">Tech</a> </div>
<div class="entry-content description clearfix">
<p>Deploying code on a Friday is a hot button topic that pops up in tech every few months, setting twitter and the like ablaze with passionate discourse in both directions. &#8220;It&#8217;s too risky to on call folks&#8221;, &#8220;no, it&#8217;s perfectly fine if you build your systems right&#8221; and so on. This is me adding fuel to that fire.</p>
<p>Some first principles to begin with:</p>
<ol>
<li>A non-zero number of deploys will go wrong. <em>No successful company in the world has ever existed or will ever exist that doesn&#8217;t face failure often</em>.</li>
<li>We don&#8217;t know with absolutely certainty before taking any action that it will guarantee success. In fact, we might not know it wasn&#8217;t successful immediately after (sometimes not until long after).</li>
<li>No one wants to spend more time working on a problem past the end of a working day than they have to, especially into extended time off like a weekend.</li>
</ol>
<p>If you deploy at all on Fridays, it will eventually require you to work late because of something unforeseen, against the interest of you and any involved teammates.</p>
<p>This may seem as though I&#8217;m advocating for the &#8220;No Friday Deploys&#8221; camp, but just the opposite.</p>
<p>Friday deploys will happen whether you want them to or not. Extenuating circumstances will at some point demand that you deploy during undesirable times (&#8220;We have a SEV0 security patch&#8221;, &#8220;We&#8217;re launching an experiment that needs to run on Saturdays and Sundays&#8221;, etc.) , where the cost for failure may be higher. If that&#8217;s the case, a mature engineering organization must be well practiced to do so.</p>
<h2 class="wp-block-heading">&#8220;Friday Deploys are Resilience&#8221;</h2>
<p>I didn&#8217;t originally say that, but I wish I did. I want it on a laptop sticker and maybe a tee. Originally, <a rel="noreferrer noopener" aria-label="Ryan Kitchens (opens in a new tab)" href="https://twitter.com/this_hits_home" target="_blank">Ryan Kitchens</a> tweeted that out as a snarky response to a <a rel="noreferrer noopener" aria-label="thread on twitter rehashing the Friday deploy debate (opens in a new tab)" href="https://twitter.com/this_hits_home/status/1164694784134373376?s=21" target="_blank">thread on twitter rehashing the Friday deploy debate</a> because apparently it had been too long since the last time we argued about this and poking the bear is just too much fun.</p>
<p>As a quick reminder, Resilience is the demonstration of <a rel="noreferrer noopener" aria-label="Graceful Extensibility (opens in a new tab)" href="https://www.researchgate.net/publication/327427067_The_Theory_of_Graceful_Extensibility_Basic_rules_that_govern_adaptive_systems" target="_blank">Graceful Extensibility</a>, that is, our ability to adapt when some capacity to work is exceeded. Human intervention is required to sort things out. This is the scary part of engineering, but also the part that makes it feel like we&#8217;re breaking new ground. After all, if we had all the instructions, it would just be data entry.</p>
<p>The only way Resilience can happen, though, is through experience, often when things go wrong. Yes, we need things to go wrong to help us understand when things go wrong. Everyone has their story about the unexpected and a long drawn out incident or a particularly painful, high stress situation where the company&#8217;s future dances on the edge of a knife. We dread when they happen but are grateful for their teachings later.</p>
<h2 class="wp-block-heading">Getting Good when Breaking Bad</h2>
<p>We should be able to deploy on a Friday. In fact, we <em>have to</em> deploy on Fridays. It&#8217;s not enough to say it&#8217;s ok to do it when needed in extreme or unusual circumstances. It needs to be done regularly and with thoughtful acknowledgement of the scope, both with the comfort of the banal and &#8220;The Fear&#8221; of any high stakes deploy going south. I would even venture to say Friday deploys should be a requirement of any functional engineering organization that exists in a universe with Fridays.</p>
<p>There&#8217;s always a choice, but it may not be palatable. At some point, you&#8217;re going to have to deploy on a Friday, for business necessity, for security concerns, something out of the ordinary that will break your cadence of what you prefer your deploy schedule to be like. This has many similar tones to the philosophy behind <a rel="noreferrer noopener" aria-label="Chaos Engineering (opens in a new tab)" href="https://principlesofchaos.org/?lang=ENcontent" target="_blank">Chaos Engineering</a>, that failure will happen, so we should minimize the blast radius where possible and understand how things will go wrong. Be prepared for failure in what should be a common scenario.</p>
<p>I&#8217;d liken deploying on a Friday to testing your datastore backups. Like the adage goes, an untested backup is no backup at all. Can you be confident that when disaster strikes, your data retention scheme can withstand losing a primary and failing over to a replica? We can, should, and must explicitly view deploys in varying time periods and circumstances as a necessity for unforeseen circumstances. Develop a frequently practiced plan for Friday deploys so that when you <em>must</em> do so, the boundaries are better understood.</p>
<p>There is nothing inherently riskier about deploying on a Friday than any other day of the week. The cost of a &#8220;surprise&#8221; may be higher, but that does not mean the decision carries with it a greater possibility of failure inherent to the decision. What <em>does</em> make the change on a Friday riskier is not doing it often enough. We should know what we&#8217;re in for when we collectively agree that yes it&#8217;s not ideal but the decision to not make a change &#8211; to not deploy on a Friday &#8211; comes with a greater cost than the known (an important qualifier &#8211; <em>it&#8217;s not fully known!</em>) potential cost of failure.</p>
<p>Resilience is preparing for this eventuality, that you will have to deploy on a Friday.</p>
<h2 class="wp-block-heading">But&#8230;not too regularly&#8230;</h2>
<p>But don&#8217;t wake me up in the middle of the night, or take me away from dinner with my family. No one wants that either. I&#8217;ve got things to do on a Friday night (ok, it&#8217;s listening to podcasts and putting together jigsaw puzzles). We <em>need</em> personal time to recharge, to be ready for the rest of the work week. Our resilience to the unexpected is also affected by fatigue and that has to be taken into consideration as well. When disaster strikes at times that cost us energy to pay down, and less to have during working hours, no one&#8217;s happy.</p>
<p>Just because you <em>can</em> deploy on a Friday doesn&#8217;t mean you <em>have to.</em> The practice should provoke a deliberate testing of your systems, human and machine alike. We&#8217;re not deploying at 4:59pm and rushing out the door because we&#8217;re too impatient to wait until Monday morning. That&#8217;s the reasonable pushback from opponents of Friday deploys, and I wholeheartedly agree.</p>
<p>The issue here is control. We&#8217;re afraid of Friday deploys because we don&#8217;t know what will happen, or more so we imagine what <em>could</em> happen and run as far away from it as possible. This control (&#8220;No Friday deploys!&#8221;) is simplistic and fragile, with the mistaken belief that if we just <em>say</em> we won&#8217;t deploy then, we&#8217;ll never have to. Communicate early and often, execute with preparation for the unknown, and remove the dread behind it. </p>
<p>Live a little bit in that uncomfortable area of uncertainty and make sure it doesn&#8217;t become unwieldy. Keep deploying on Fridays to verify they&#8217;re sound. Just not too often.</p>
<p style="text-align:left"><em>Thanks to <a rel="noreferrer noopener" aria-label=" (opens in a new tab)" href="https://twitter.com/jhscott" target="_blank">Jacob Scott </a>for the inspiration for the title and <a rel="noreferrer noopener" aria-label="Ryan Kitchens (opens in a new tab)" href="https://twitter.com/this_hits_home" target="_blank">Ryan Kitchens</a> for nerdsniping me to write this.</em></p>
<p style="text-align:right"><br><em>Photo: </em><a rel="noreferrer noopener" aria-label="https://www.flickr.com/photos/oneaustin/1261907803 (opens in a new tab)" href="https://www.flickr.com/photos/oneaustin/1261907803/" target="_blank">https://www.flickr.com/photos/oneaustin/1261907803</a><a href="https://www.flickr.com/photos/oneaustin/1261907803/"></a>/</p>
</div><!-- .entry-content -->
<footer class="clearfix">
<p class="tags"><i class="fa fa-tags"></i> <span>Tags:</span><a href="https://willgallego.com/tag/deploys/" rel="tag">deploys</a> <a href="https://willgallego.com/tag/resilience-engineering/" rel="tag">Resilience Engineering</a></p> </footer><!-- .entry -->
</article><!-- #post-449 -->
<div id="posts-pagination" class="clearfix">
<h3 class="sr-only">Post navigation</h3>
<div class="previous pull-left"><a href="https://willgallego.com/2019/07/13/pursuing-excellence-not-perfection/" rel="prev">&larr; Pursuing Excellence, not Perfection</a></div>
<div class="next pull-right"><a href="https://willgallego.com/2019/12/07/patience-in-implementing-effective-incident-reviews/" rel="next">Patience in Implementing Effective Incident Reviews &rarr;</a></div>
</div><!-- #posts-pagination -->
<div id="comments" class="comments-area">
<h2 id="comments-title">
<i class="fa fa-comments"></i>&nbsp;
5 comments for &ldquo;<span>Continuous Verification of Friday Deploys</span>&rdquo; </h2>
<ol class="commentlist">
<li id="comment-340" class="pingback">
<div class="comment-body">
<i class="fa fa-paperclip"></i>
Pingback: <a href="http://sreweekly.com/sre-weekly-issue-182/" class="url" rel="ugc external nofollow">SRE Weekly Issue #182 &#8211; SRE WEEKLY</a> </div>
</li><!-- #comment-## -->
<li id="comment-355" class="pingback">
<div class="comment-body">
<i class="fa fa-paperclip"></i>
Pingback: <a href="https://www.jvt.me/mf2/251021ee-0fc8-4f8a-a9f0-8a65ff96a6f4/" class="url" rel="ugc external nofollow">&middot; Jamie Tanna | Software (Quality) Engineer</a> </div>
</li><!-- #comment-## -->
<li id="comment-397" class="pingback">
<div class="comment-body">
<i class="fa fa-paperclip"></i>
Pingback: <a href="https://www.jvt.me/mf2/2019/08/4qj2z/" class="url" rel="ugc external nofollow">Continuous Verification of Friday Deploys &middot; Jamie Tanna | Software (Quality) Engineer</a> </div>
</li><!-- #comment-## -->
<li id="comment-621" class="pingback">
<div class="comment-body">
<i class="fa fa-paperclip"></i>
Pingback: <a href="https://willgallego.com/2021/08/14/rethinking-best-practices/" class="url" rel="ugc">Rethinking Best Practices &#8211; Will Gallego</a> </div>
</li><!-- #comment-## -->
<li id="comment-626" class="pingback">
<div class="comment-body">
<i class="fa fa-paperclip"></i>
Pingback: <a href="https://e-gram.in/2021/12/12/rethinking-best-practices-will-gallego/" class="url" rel="ugc external nofollow">Rethinking Best Practices – Will Gallego &#8211; My Blog</a> </div>
</li><!-- #comment-## -->
</ol><!-- .commentlist -->
<p class="nocomments">Comments are closed.</p>
</div><!-- #comments .comments-area -->
</div>
<div id="secondary" class="col-md-4" role="complementary">
<aside id="recent-posts-4" class="widget widget_recent_entries">
<h3 class="widget-title">Recent Posts</h3>
<ul>
<li>
<a href="https://willgallego.com/2025/06/21/big-enough-to-fail/">Big Enough to Fail</a>
</li>
<li>
<a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/">An Incident Review of an Incident Review</a>
</li>
<li>
<a href="https://willgallego.com/2024/11/11/job-switching-and-titles/">Job Switching and Titles</a>
</li>
<li>
<a href="https://willgallego.com/2024/07/12/the-fridge-theory-of-blameless-organizations/">The Fridge Theory of Blameless Organizations</a>
</li>
</ul>
</aside><aside id="search-2" class="widget widget_search"><form role="search" method="get" class="search-form" action="https://willgallego.com/">
<label>
<span class="screen-reader-text">Search for:</span>
<input type="search" class="search-field" placeholder="Search &hellip;" value="" name="s" />
</label>
<input type="submit" class="search-submit" value="Search" />
</form></aside>
<aside id="recent-posts-2" class="widget widget_recent_entries">
<h3 class="widget-title">Recent Posts</h3>
<ul>
<li>
<a href="https://willgallego.com/2025/06/21/big-enough-to-fail/">Big Enough to Fail</a>
</li>
<li>
<a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/">An Incident Review of an Incident Review</a>
</li>
<li>
<a href="https://willgallego.com/2024/11/11/job-switching-and-titles/">Job Switching and Titles</a>
</li>
<li>
<a href="https://willgallego.com/2024/07/12/the-fridge-theory-of-blameless-organizations/">The Fridge Theory of Blameless Organizations</a>
</li>
<li>
<a href="https://willgallego.com/2024/06/21/how-many-maybes-until-empathy/">How Many Maybe&#8217;s until Empathy?</a>
</li>
</ul>
</aside><aside id="recent-comments-2" class="widget widget_recent_comments"><h3 class="widget-title">Recent Comments</h3><ul id="recentcomments"><li class="recentcomments"><span class="comment-author-link"><a href="https://www.arnoldit.com/wordpress/2025/12/04/titanic-talk-this-ship-will-not-fail/" class="url" rel="ugc external nofollow">Titanic Talk: This Ship Will Not Fail : Stephen E. Arnold @ Beyond Search</a></span> on <a href="https://willgallego.com/2025/06/21/big-enough-to-fail/#comment-37339">Big Enough to Fail</a></li><li class="recentcomments"><span class="comment-author-link"><a href="https://sreweekly.com/sre-weekly-issue-484/" class="url" rel="ugc external nofollow">SRE Weekly Issue #484 &#8211; SRE WEEKLY</a></span> on <a href="https://willgallego.com/2025/06/21/big-enough-to-fail/#comment-25648">Big Enough to Fail</a></li><li class="recentcomments"><span class="comment-author-link">Jacob Rose</span> on <a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/#comment-17389">An Incident Review of an Incident Review</a></li><li class="recentcomments"><span class="comment-author-link">Leonardo Camara</span> on <a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/#comment-16366">An Incident Review of an Incident Review</a></li><li class="recentcomments"><span class="comment-author-link"><a href="https://sreweekly.com/sre-weekly-issue-460/" class="url" rel="ugc external nofollow">SRE Weekly Issue #460 &#8211; SRE WEEKLY</a></span> on <a href="https://willgallego.com/2025/01/11/an-incident-review-of-an-incident-review/#comment-16284">An Incident Review of an Incident Review</a></li></ul></aside><aside id="archives-2" class="widget widget_archive"><h3 class="widget-title">Archives</h3>
<ul>
<li><a href='https://willgallego.com/2025/06/'>June 2025</a></li>
<li><a href='https://willgallego.com/2025/01/'>January 2025</a></li>
<li><a href='https://willgallego.com/2024/11/'>November 2024</a></li>
<li><a href='https://willgallego.com/2024/07/'>July 2024</a></li>
<li><a href='https://willgallego.com/2024/06/'>June 2024</a></li>
<li><a href='https://willgallego.com/2024/03/'>March 2024</a></li>
<li><a href='https://willgallego.com/2024/02/'>February 2024</a></li>
<li><a href='https://willgallego.com/2024/01/'>January 2024</a></li>
<li><a href='https://willgallego.com/2023/10/'>October 2023</a></li>
<li><a href='https://willgallego.com/2023/04/'>April 2023</a></li>
<li><a href='https://willgallego.com/2023/01/'>January 2023</a></li>
<li><a href='https://willgallego.com/2022/12/'>December 2022</a></li>
<li><a href='https://willgallego.com/2022/06/'>June 2022</a></li>
<li><a href='https://willgallego.com/2022/02/'>February 2022</a></li>
<li><a href='https://willgallego.com/2022/01/'>January 2022</a></li>
<li><a href='https://willgallego.com/2021/08/'>August 2021</a></li>
<li><a href='https://willgallego.com/2021/05/'>May 2021</a></li>
<li><a href='https://willgallego.com/2021/02/'>February 2021</a></li>
<li><a href='https://willgallego.com/2021/01/'>January 2021</a></li>
<li><a href='https://willgallego.com/2020/09/'>September 2020</a></li>
<li><a href='https://willgallego.com/2020/06/'>June 2020</a></li>
<li><a href='https://willgallego.com/2020/05/'>May 2020</a></li>
<li><a href='https://willgallego.com/2020/02/'>February 2020</a></li>
<li><a href='https://willgallego.com/2019/12/'>December 2019</a></li>
<li><a href='https://willgallego.com/2019/08/'>August 2019</a></li>
<li><a href='https://willgallego.com/2019/07/'>July 2019</a></li>
<li><a href='https://willgallego.com/2019/04/'>April 2019</a></li>
<li><a href='https://willgallego.com/2019/02/'>February 2019</a></li>
<li><a href='https://willgallego.com/2018/12/'>December 2018</a></li>
<li><a href='https://willgallego.com/2018/08/'>August 2018</a></li>
<li><a href='https://willgallego.com/2018/07/'>July 2018</a></li>
<li><a href='https://willgallego.com/2018/04/'>April 2018</a></li>
<li><a href='https://willgallego.com/2018/02/'>February 2018</a></li>
<li><a href='https://willgallego.com/2018/01/'>January 2018</a></li>
<li><a href='https://willgallego.com/2017/11/'>November 2017</a></li>
<li><a href='https://willgallego.com/2017/10/'>October 2017</a></li>
<li><a href='https://willgallego.com/2017/09/'>September 2017</a></li>
</ul>
</aside> </div><!-- #secondary.widget-area --> </div>
</div>
</main><!-- main -->
<footer id="footer" role="contentinfo">
<div id="footer-content" class="container">
<div class="row">
<div class="copyright col-lg-12">
<span class="pull-left">Copyright &copy; 2026 <a href="https://willgallego.com">Will Gallego</a>. All Rights Reserved.</span>
<span class="credit-link pull-right"><i class="fa fa-leaf"></i>The Arcade Basic Theme by <a href="https://themes.bavotasan.com/themes/arcade">bavotasan.com</a>.</span>
</div><!-- .col-lg-12 -->
</div><!-- .row -->
</div><!-- #footer-content.container -->
</footer><!-- #footer -->
</div><!-- #page -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"\/*"},{"not":{"href_matches":["\/wp-*.php","\/wp-admin\/*","\/wp-content\/uploads\/*","\/wp-content\/*","\/wp-content\/plugins\/*","\/wp-content\/themes\/arcade-basic\/*","\/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script type="text/javascript" src="https://willgallego.com/wp-includes/js/comment-reply.min.js?ver=6.8.8" id="comment-reply-js" async="async" data-wp-strategy="async"></script>
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/bootstrap.min.js?ver=3.0.3" id="bootstrap-js"></script>
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/fillsize.js?ver=6.8.8" id="fillsize-js"></script>
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/jquery.arctext.js?ver=6.8.8" id="arctext-js"></script>
<script type="text/javascript" id="arcade-basic-script-js-extra">
/* <![CDATA[ */
var arcade_basic_vars = {"arc":"800","fittext":""};
/* ]]> */
</script>
<script type="text/javascript" src="https://willgallego.com/wp-content/themes/arcade-basic/library/js/theme.js?ver=6.8.8" id="arcade-basic-script-js"></script>
</body>
</html>
<!-- Page cached by LiteSpeed Cache 7.5.0.1 on 2026-09-11 17:17:00 -->