Files
nexus/sreweekly/articles/394/01-creating-checklists-for-high-stakes-changes.html
2026-09-12 17:23:01 +08:00

96 lines
20 KiB
HTML

<!doctype html><html lang=en><head><link href=https://gmpg.org/xfn/11 rel=profile><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=viewport content="width=device-width,initial-scale=1"><title>Creating Checklists for High Stakes Changes — Nick Janetakis</title><meta name=description content="The use case we'll go over is performing a major database upgrade for a large application that's running in production."><meta name=author content="Nick Janetakis"><link rel=canonical href=https://nickjanetakis.com/blog/creating-checklists-for-high-stakes-changes><link rel=sitemap type=application/xml title=Sitemap href=/sitemap.xml><link rel=alternate type=application/rss+xml title="RSS Feed for https://nickjanetakis.com/" href=/atom.xml><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="https://nickjanetakis.com/assets/blog/cards/creating-checklists-for-high-stakes-changes.jpg"><meta name=twitter:site content="@nickjanetakis"><meta name=twitter:creator content="@nickjanetakis"><meta name=twitter:title content="Creating Checklists for High Stakes Changes — Nick Janetakis"><meta name=twitter:description content="The use case we'll go over is performing a major database upgrade for a large application that's running in production."><meta property="og:locale" content="en_us"><meta property="og:type" content="article"><meta property="article:published_time" content="2023-06-27T00:00:00+00:00"><meta property="og:title" content="Creating Checklists for High Stakes Changes — Nick Janetakis"><meta property="og:description" content="The use case we'll go over is performing a major database upgrade for a large application that's running in production."><meta property="og:url" content="https://nickjanetakis.com/blog/creating-checklists-for-high-stakes-changes"><meta property="og:site_name" content="Nick Janetakis"><meta property="og:image" content="https://nickjanetakis.com/assets/blog/cards/creating-checklists-for-high-stakes-changes.jpg"><link rel=icon href=/favicon.ico sizes=16x16><link rel="shortcut icon" href=/favicon.png><link rel=apple-touch-icon href=/favicon-57.png sizes=57x57><link rel=apple-touch-icon href=/favicon-72.png sizes=72x72><link rel=apple-touch-icon href=/favicon-76.png sizes=76x76><link rel=apple-touch-icon href=/favicon-114.png sizes=114x114><link rel=apple-touch-icon href=/favicon-120.png sizes=120x120><link rel=apple-touch-icon href=/favicon-144.png sizes=144x144><link rel=apple-touch-icon href=/favicon-152.png sizes=152x152><link rel=apple-touch-icon href=/favicon-180.png sizes=180x180><link href="//fonts.googleapis.com/css?family=Open+Sans:400,400italic,600,700" rel=stylesheet type=text/css><link rel=stylesheet href="/assets/css/app.bc7ea1444e731381cdcb4bee8a406f702ef02ac3807d91ae14831e77a0925e96.css" integrity="sha256-vH6hRE5zE4HNy0vuikBvcC7wKsOAfZGuFIMed6CSXpY="><script src=https://cdnjs.cloudflare.com/ajax/libs/jquery/1.12.4/jquery.min.js></script><script>(function(e,t,n,s,o,i,a){e.GoogleAnalyticsObject=o,e[o]=e[o]||function(){(e[o].q=e[o].q||[]).push(arguments)},e[o].l=1*new Date,i=t.createElement(n),a=t.getElementsByTagName(n)[0],i.async=1,i.src=s,a.parentNode.insertBefore(i,a)})(window,document,"script","//www.google-analytics.com/analytics.js","ga"),ga("create","UA-56321149-1","auto"),ga("require","displayfeatures"),ga("send","pageview")</script></head><body><nav class="navbar navbar-white navbar-fixed-top"><div class=container><div class="col-lg-offset-2 col-lg-8"><div class=navbar-header><button type=button class="navbar-toggle collapsed" data-toggle=collapse data-target=#bs-navbar-collapse aria-expanded=false>
<span class=sr-only>Toggle navigation</span>
<span class=icon-bar></span>
<span class=icon-bar></span>
<span class=icon-bar></span>
</button>
<a class=navbar-brand href=/>Nick Janetakis</a></div><div class="collapse navbar-collapse" id=bs-navbar-collapse><ul class="nav navbar-nav navbar-right"><li><a href=/courses><span>Courses</span></a></li><li><a href=/blog class=active>Blog</a></li><li><a href=/podcast>Podcast</a></li><li><a href=/about>About</a></li><li><a href=/work-together>Work Together</a></li><li><a href=/newsletter>Get Updates</a></li></ul></div></div></div></nav><div class=cta-banner><div class=container><div class="col-lg-offset-2 col-lg-8"><h3 class=cta-banner-header>Learn Docker With My Newest Course</h3><p class=remove-margin-bottom><a class=cta-banner-link href="https://diveintodocker.com/?utm_source=nj&utm_medium=website-banner&utm_campaign=%2fblog%2fcreating-checklists-for-high-stakes-changes">Dive into Docker</a> takes you from "What is Docker?" to confidently applying Docker to your own projects. It's packed with best practices and examples.
<a class=cta-banner-link href="https://diveintodocker.com/?utm_source=nj&utm_medium=website-banner&utm_campaign=%2fblog%2fcreating-checklists-for-high-stakes-changes">Start Learning Docker &rarr;</a></p></div></div></div><div class="social-share-floating-bar hidden-sm hidden-xs"><a class=share-btn-twitter rel=noopener href="https://x.com/intent/tweet?url=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&text=Creating%20Checklists%20for%20High%20Stakes%20Changes%20by%20%40nickjanetakis"><i class="fa fa-twitter"></i>
</a><a class=share-btn-facebook rel=noopener href="https://www.facebook.com/sharer/sharer.php?u=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&title=Creating%20Checklists%20for%20High%20Stakes%20Changes"><i class="fa fa-facebook"></i>
</a><a class=share-btn-linkedin rel=noopener href="http://www.linkedin.com/shareArticle?mini=true&url=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&title=Creating%20Checklists%20for%20High%20Stakes%20Changes"><i class="fa fa-linkedin"></i></a></div><div class=container><div class="col-lg-offset-2 col-lg-8 page"><article><p class=post-meta title="Orignially published on June 27, 2023
">Updated on June 27, 2023
in
<a href=/blog/tag/dev-mindset-tips-tricks-and-tutorials>#dev-mindset</a></p><h1 class="post h2" title="1,462 words">Creating Checklists for High Stakes Changes</h1><p><img src=/assets/blog/cards/creating-checklists-for-high-stakes-changes-18f7fbdabbcae8b5d760b4ebb02bd8b055627cacef96240a76d20d3eb513ac74.jpg class=post-card height=422 width=750 alt=creating-checklists-for-high-stakes-changes.jpg></p><h2 class="lead no-letterspacing">The use case we'll go over is performing a major database upgrade for a large application that's running in production.</h2><div class=post-quick-jump><strong>Quick Jump:</strong><nav id=TableOfContents><ul><li><ul><li><a href=#database-upgrade-workflow>Database Upgrade Workflow</a></li><li><a href=#video-version-of-this-post>Video Version of This Post</a></li></ul></li></ul></nav></div><p><strong>Prefer video? <a href=#video-version-of-this-post>This video</a> covers what&rsquo;s in this
post with a little bit more detail.</strong></p><p>Checklists can be used for more than just database upgrades of course but I&rsquo;m a
big fan of using practical examples to demonstrate something and I recently
performed a checklist driven database upgrade.</p><p>There&rsquo;s a reason airline pilots use checklists even if they have 40+ years of
experience and have taken off and landed thousands of times. Having the
workflow in front of your face lets you focus on executing the steps instead of
trying to think about the workflow while executing the steps at the same time.</p><p>For one of my clients we were preparing a major database version upgrade (MySQL
5.7 to 8.0) and we wanted to minimize downtime and what could go wrong.</p><p>Needless to say a checklist was invaluable.</p><p>It let us do all of the hard work of defining the workflow while there&rsquo;s no
pressure well before the time when the upgrade was happening. We also got a
chance to test the workflow in a test environment before applying it to the
live database.</p><p><a name=#database-upgrade-workflow></a></p><h3 id=database-upgrade-workflow><a href=#database-upgrade-workflow style=position:absolute;left:-12px>#</a>
Database Upgrade Workflow</h3><p>It was on AWS using RDS which is AWS&rsquo; managed MySQL / Postgres service but
similar steps could be used for other database upgrades too. After all of the
steps are listed out we&rsquo;ll go over the details and my thought process of each
one in the next section.</p><p><em>None of the links below go anywhere. I added them here because in reality
these led to the exact resource being acted on in a step. This helps avoid
human error and needing to click around and find things on the fly. It&rsquo;s a
minor but useful detail in my opinion.</em></p><ol><li>Ensure you can connect to the live database before doing anything</li><li>Confirm the SQL user has all necessary permissions to run any queries afterwards</li><li>Create <a href=#>maintenance page message</a> for example.com</li><li><a href=#>Enable maintenance page</a> at the firewall level</li><li><a href=#>Verify maintenance page</a> is being served for example.com</li><li>Create manual <a href=#>database snapshot</a> as a backup:<ul><li>Name it <code>example-pre-mysql8-upgrade-2023-05-08</code></li></ul></li><li><a href=#>Modify</a> the live <code>example</code> database:<ul><li>Set the DB engine to MySQL <code>8.0.32</code></li><li>Set the parameter group to <code>custom-mysql80-v1</code></li></ul></li><li>Wait until RDS is updated while <a href=#>looking at the status</a><ul><li>It may take 10-20 minutes</li></ul></li><li>Connect to the live database and run SQL queries to verify it works:<ul><li><code>SELECT ...</code> and expect <code>...</code></li><li><code>SELECT ...</code> and expect <code>...</code></li><li><code>SELECT ...</code> and expect <code>...</code></li></ul></li><li>Disable <a href=#>maintenance page</a></li><li>Verify <a href=#>example.com</a> is loading successfully</li><li>Create a calendar reminder to delete the <a href=#>manual snapshot</a> in 30 days</li><li>Notify the business that the site is back up and operational</li></ol><h4>Explanation</h4><p>Here&rsquo;s my thought process for defining the checklist the way it was:</p><p><strong>Steps 1 and 2</strong> are critically important because you could get to step 9 and
then try to run a query only to find out that the DB user you&rsquo;re connecting
with doesn&rsquo;t have permissions.</p><p>For example, our &ldquo;app&rdquo; user can&rsquo;t create or drop tables but our &ldquo;migrate&rdquo; user
can.</p><p>This happened in our test environment and it caused a blocker to obtain the
correct permissions because the person performing the upgrade (which was me in
this case) didn&rsquo;t have access to modify SQL user permissions.</p><p>Step 1 is also nice because it&rsquo;s a good idea to ensure your system is in
a working state before you modify it.</p><p><strong>Steps 3, 4 and 5</strong> are a handy way to toggle maintenance mode without needing
to perform a code change. It&rsquo;s a firewall rule we can turn on and off on demand
for 1 or more domains.</p><p><strong>Step 6</strong> is an insurance policy in case something goes wrong we have the
latest version of our DB to ensure no data loss. That&rsquo;s why we had to turn
maintenance mode on first to ensure no new data is being written to the DB.</p><p>Technically this isn&rsquo;t flawless because in between steps 6 and 7 a background
job could modify the state of our database even with no new traffic being sent
to the site. For example any scheduled work that may happen at X time in the
future.</p><p>We have pretty robust logging and observability in place to know when this
happens along with a means to reconcile everything as needed if something went
wrong and we needed to use the snapshot.</p><p>Again, this only becomes an issue if we need to use the snapshot. If a
background job runs while the database is down during the upgrade itself that&rsquo;s
ok, the job will fail and will be auto-retried. Eventually the DB will come
back up and it&rsquo;ll work.</p><p><strong>Step 7</strong> is mainly clicking buttons in a web UI. That custom parameter group
was created ahead of time in preparation for the upgrade.</p><p>Ideally all of these &ldquo;click buttons in web UI&rdquo; steps could be put into code
using Terraform so they all become independent pull requests that can be
reviewed but this client has a 10+ year old app and not all of their
infrastructure has been ported into Terraform yet.</p><p><strong>Step 8</strong> is a waiting game. The AWS config tab for your RDS instance will
show the status, such as if it&rsquo;s applying the new parameter group or if it&rsquo;s in
the process of rebooting. The time range is pretty sporadic, I noticed upwards
of a 2x difference in total time based on nothing we could control.</p><p><strong>Step 9</strong> are a few sanity check queries to ensure things updated to their
correct version and certain tables are cleared based on what the dev team
suggested.</p><p><strong>Steps 10 and 11</strong> allow the site to be live again within a few seconds of
enabling it. Automated smoke tests would be a way to automate step 11 a bit but
for now a human went to a couple of pages to make sure they worked.</p><p><strong>Step 12</strong> is important because a snapshot costs money to store. We wanted to
keep it as a snapshot to quickly restore it as needed. We could have done a SQL
dump and stored it on S3 but realistically spending a couple of bucks to keep
the snapshot around is worth it.</p><p>If something goes wrong it lets the developers quickly spin up a test DB based
off the snapshot to help figure out what&rsquo;s happening and address the situation.</p><p><strong>Step 13</strong> is always nice to let stakeholders know how things went. In our
case everything went flawlessly and we had around 15 minutes of expected
downtime which was done during a time where our traffic usually dips. The
preparation work paid off.</p><hr><p>You&rsquo;ll notice there&rsquo;s no steps or workflow for if something goes wrong. That&rsquo;s
because I both wrote and executed the steps. At this point I was ok with
letting my overall general experience guide troubleshooting anything that might
go wrong.</p><p>If I were writing these steps for someone else, that would be different. I&rsquo;d
add more details.</p><p>I also executed them in a test environment multiple times to work out the
kinks. By the time it was executed in production I was feeling really good
about it.</p><p>We did have a disaster recovery plan though. It wasn&rsquo;t a formal document but we
thought about what we could do if certain things went wrong. Realistically
steps 7 and 8 are the riskiest ones and it&rsquo;s technically out of our control.
Either RDS will successfully do it or not.</p><h4>While it&rsquo;s Happening</h4><p>While executing the steps I created screenshots and quickly jotted down how
long each step took. I really like this idea because once the whole process is
finished you can go back and make a little post with a timeline that mentions
anything interesting that may have happened along the way.</p><p>This really helps your future self or someone else understand the process in
case they need to do a similar thing in the future. When performing a scary
upgrade, having the extra context and details is well worth the 15 minutes
writing up a summary when it happened.</p><p><a name=#video-version-of-this-post></a></p><h3 id=video-version-of-this-post><a href=#video-version-of-this-post style=position:absolute;left:-12px>#</a>
Video Version of This Post</h3><div style=position:relative;padding-bottom:56.25%;height:0;overflow:hidden><iframe allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share; fullscreen" loading=eager referrerpolicy=strict-origin-when-cross-origin src="https://www.youtube.com/embed/hOp7uog7_Xw?autoplay=0&amp;controls=1&amp;end=0&amp;loop=0&amp;mute=0&amp;start=0" style=position:absolute;top:0;left:0;width:100%;height:100%;border:0 title="YouTube video"></iframe></div><p></p><h4>Timestamps</h4><ul><li>0:44 &ndash; Checklists are valuable</li><li>2:09 &ndash; Going over the database upgrade checklist</li><li>3:26 &ndash; Ensure you can connect to your DB</li><li>4:27 &ndash; Turning on maintenance mode for the site</li><li>5:55 &ndash; Create a manual snapshot as a backup and perform the upgrade</li><li>10:49 &ndash; Confirm it works with a few SQL queries</li><li>11:30 &ndash; Turning off maintenance mode for the site</li><li>12:10 &ndash; Create a calendar reminder to delete the snapshot backup</li><li>13:53 &ndash; Notifying the business that everything went well</li><li>14:24 &ndash; What if something went wrong?</li><li>16:30 &ndash; Making a timeline of interesting details afterwards</li><li>17:39 &ndash; What was the last thing you made a checklist for?</li></ul><p><strong>What was the last thing you made a checklist for? Let me know below.</strong></p></article><div class="cta-header-with-optin cta-header-with-optin-dark text-center"><h4>Never Miss a Tip, Trick or Tutorial</h4><form id=signup method=post class=form-horizontal><input type=hidden name=referrer value=https://nickjanetakis.com/blog/creating-checklists-for-high-stakes-changes>
<input type=hidden class=gmt name=gmt value=-4>
<input type=hidden name=list value=Hndxczl9v0YCEzbZ3VuVkg>
<input type=hidden name=subform value=yes><div class="form-group form-full-name" aria-hidden=true><div class="col-md-6 col-md-offset-3"><label class=sr-only for=full-name>Do not fill this field out as a human</label>
<input type=text id=full-name name=hp class=form-control placeholder="Full Name" tabindex=-1 autocomplete=off></div></div><div class=form-group><div class="col-md-6 col-md-offset-3"><label class=sr-only for=email>Email Address</label><div class=input-group><input type=email id=email name=email class=form-control placeholder="Email Address"><div class=input-group-btn><button type=submit class="btn btn-warning">
<strong>Get Updates</strong></button></div></div></div></div></form></div><p class="text-muted small half-margin-top text-center">Like you, I'm super protective of my inbox,
so don't worry about getting spammed. You can expect a few emails per year (at most), and
you can 1-click unsubscribe at any time. <a href=/newsletter>See what else you'll get</a> too.</p><hr><div class=social-share-btns-container><div class=social-share-btns><a class="share-btn share-btn-twitter" rel=noopener href="https://x.com/intent/tweet?url=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&text=Creating%20Checklists%20for%20High%20Stakes%20Changes%20by%20%40nickjanetakis"><i class="fa fa-twitter"></i>
Share
</a><a class="share-btn share-btn-facebook" rel=noopener href="https://www.facebook.com/sharer/sharer.php?u=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&title=Creating%20Checklists%20for%20High%20Stakes%20Changes"><i class="fa fa-facebook"></i>
Share
</a><a class="share-btn share-btn-linkedin" rel=noopener href="http://www.linkedin.com/shareArticle?mini=true&url=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&title=Creating%20Checklists%20for%20High%20Stakes%20Changes"><i class="fa fa-linkedin"></i>
Share</a></div></div><hr><h2>Comments</h2><div id=disqus_thread data-permalink=https://nickjanetakis.com/blog/creating-checklists-for-high-stakes-changes data-relpermalink=/blog/creating-checklists-for-high-stakes-changes></div><script>var disqus_config=function(){var e=document.querySelector("#disqus_thread");this.page.url=e.dataset.permalink,this.page.identifier=e.dataset.relpermalink};(function(){var e=document,t=e.createElement("script");t.src="//nickjj.disqus.com/embed.js",t.setAttribute("data-timestamp",+new Date),(e.head||e.body).appendChild(t)})()</script><noscript>Please enable JavaScript to view the <a href=https://disqus.com/?ref_noscript rel=nofollow>comments powered by Disqus.</a></noscript><script id=dsq-count-scr src=//nickjj.disqus.com/count.js async></script><hr><footer><p>&copy; 2026 Nick Janetakis</p><a target=_blank class="small twitter-follow-button" href=https://x.com/nickjanetakis><span class=text-primary>Follow @nickjanetakis</span></a><p class="small tiny-margin-top"><a target=_blank href=https://github.com/nickjj><span class="fa fa-github text-black"></span> GitHub
</a>|
<a target=_blank href=https://www.youtube.com/c/nickjanetakis><span class="fa fa-youtube text-black"></span> YouTube</a></p></footer></div></div><script type=text/javascript src=/assets/js/app.9b2ef33b81dccfc0bf4a638a2b2041902827deb5ad23846d48c1a506bc6b21c6.js></script></body></html>