96 lines
20 KiB
HTML
96 lines
20 KiB
HTML
<!doctype html><html lang=en><head><link href=https://gmpg.org/xfn/11 rel=profile><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=viewport content="width=device-width,initial-scale=1"><title>Creating Checklists for High Stakes Changes — Nick Janetakis</title><meta name=description content="The use case we'll go over is performing a major database upgrade for a large application that's running in production."><meta name=author content="Nick Janetakis"><link rel=canonical href=https://nickjanetakis.com/blog/creating-checklists-for-high-stakes-changes><link rel=sitemap type=application/xml title=Sitemap href=/sitemap.xml><link rel=alternate type=application/rss+xml title="RSS Feed for https://nickjanetakis.com/" href=/atom.xml><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="https://nickjanetakis.com/assets/blog/cards/creating-checklists-for-high-stakes-changes.jpg"><meta name=twitter:site content="@nickjanetakis"><meta name=twitter:creator content="@nickjanetakis"><meta name=twitter:title content="Creating Checklists for High Stakes Changes — Nick Janetakis"><meta name=twitter:description content="The use case we'll go over is performing a major database upgrade for a large application that's running in production."><meta property="og:locale" content="en_us"><meta property="og:type" content="article"><meta property="article:published_time" content="2023-06-27T00:00:00+00:00"><meta property="og:title" content="Creating Checklists for High Stakes Changes — Nick Janetakis"><meta property="og:description" content="The use case we'll go over is performing a major database upgrade for a large application that's running in production."><meta property="og:url" content="https://nickjanetakis.com/blog/creating-checklists-for-high-stakes-changes"><meta property="og:site_name" content="Nick Janetakis"><meta property="og:image" content="https://nickjanetakis.com/assets/blog/cards/creating-checklists-for-high-stakes-changes.jpg"><link rel=icon href=/favicon.ico sizes=16x16><link rel="shortcut icon" href=/favicon.png><link rel=apple-touch-icon href=/favicon-57.png sizes=57x57><link rel=apple-touch-icon href=/favicon-72.png sizes=72x72><link rel=apple-touch-icon href=/favicon-76.png sizes=76x76><link rel=apple-touch-icon href=/favicon-114.png sizes=114x114><link rel=apple-touch-icon href=/favicon-120.png sizes=120x120><link rel=apple-touch-icon href=/favicon-144.png sizes=144x144><link rel=apple-touch-icon href=/favicon-152.png sizes=152x152><link rel=apple-touch-icon href=/favicon-180.png sizes=180x180><link href="//fonts.googleapis.com/css?family=Open+Sans:400,400italic,600,700" rel=stylesheet type=text/css><link rel=stylesheet href="/assets/css/app.bc7ea1444e731381cdcb4bee8a406f702ef02ac3807d91ae14831e77a0925e96.css" integrity="sha256-vH6hRE5zE4HNy0vuikBvcC7wKsOAfZGuFIMed6CSXpY="><script src=https://cdnjs.cloudflare.com/ajax/libs/jquery/1.12.4/jquery.min.js></script><script>(function(e,t,n,s,o,i,a){e.GoogleAnalyticsObject=o,e[o]=e[o]||function(){(e[o].q=e[o].q||[]).push(arguments)},e[o].l=1*new Date,i=t.createElement(n),a=t.getElementsByTagName(n)[0],i.async=1,i.src=s,a.parentNode.insertBefore(i,a)})(window,document,"script","//www.google-analytics.com/analytics.js","ga"),ga("create","UA-56321149-1","auto"),ga("require","displayfeatures"),ga("send","pageview")</script></head><body><nav class="navbar navbar-white navbar-fixed-top"><div class=container><div class="col-lg-offset-2 col-lg-8"><div class=navbar-header><button type=button class="navbar-toggle collapsed" data-toggle=collapse data-target=#bs-navbar-collapse aria-expanded=false>
|
|
<span class=sr-only>Toggle navigation</span>
|
|
<span class=icon-bar></span>
|
|
<span class=icon-bar></span>
|
|
<span class=icon-bar></span>
|
|
</button>
|
|
<a class=navbar-brand href=/>Nick Janetakis</a></div><div class="collapse navbar-collapse" id=bs-navbar-collapse><ul class="nav navbar-nav navbar-right"><li><a href=/courses><span>Courses</span></a></li><li><a href=/blog class=active>Blog</a></li><li><a href=/podcast>Podcast</a></li><li><a href=/about>About</a></li><li><a href=/work-together>Work Together</a></li><li><a href=/newsletter>Get Updates</a></li></ul></div></div></div></nav><div class=cta-banner><div class=container><div class="col-lg-offset-2 col-lg-8"><h3 class=cta-banner-header>Learn Docker With My Newest Course</h3><p class=remove-margin-bottom><a class=cta-banner-link href="https://diveintodocker.com/?utm_source=nj&utm_medium=website-banner&utm_campaign=%2fblog%2fcreating-checklists-for-high-stakes-changes">Dive into Docker</a> takes you from "What is Docker?" to confidently applying Docker to your own projects. It's packed with best practices and examples.
|
|
<a class=cta-banner-link href="https://diveintodocker.com/?utm_source=nj&utm_medium=website-banner&utm_campaign=%2fblog%2fcreating-checklists-for-high-stakes-changes">Start Learning Docker →</a></p></div></div></div><div class="social-share-floating-bar hidden-sm hidden-xs"><a class=share-btn-twitter rel=noopener href="https://x.com/intent/tweet?url=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&text=Creating%20Checklists%20for%20High%20Stakes%20Changes%20by%20%40nickjanetakis"><i class="fa fa-twitter"></i>
|
|
</a><a class=share-btn-facebook rel=noopener href="https://www.facebook.com/sharer/sharer.php?u=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&title=Creating%20Checklists%20for%20High%20Stakes%20Changes"><i class="fa fa-facebook"></i>
|
|
</a><a class=share-btn-linkedin rel=noopener href="http://www.linkedin.com/shareArticle?mini=true&url=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&title=Creating%20Checklists%20for%20High%20Stakes%20Changes"><i class="fa fa-linkedin"></i></a></div><div class=container><div class="col-lg-offset-2 col-lg-8 page"><article><p class=post-meta title="Orignially published on June 27, 2023
|
|
">Updated on June 27, 2023
|
|
in
|
|
<a href=/blog/tag/dev-mindset-tips-tricks-and-tutorials>#dev-mindset</a></p><h1 class="post h2" title="1,462 words">Creating Checklists for High Stakes Changes</h1><p><img src=/assets/blog/cards/creating-checklists-for-high-stakes-changes-18f7fbdabbcae8b5d760b4ebb02bd8b055627cacef96240a76d20d3eb513ac74.jpg class=post-card height=422 width=750 alt=creating-checklists-for-high-stakes-changes.jpg></p><h2 class="lead no-letterspacing">The use case we'll go over is performing a major database upgrade for a large application that's running in production.</h2><div class=post-quick-jump><strong>Quick Jump:</strong><nav id=TableOfContents><ul><li><ul><li><a href=#database-upgrade-workflow>Database Upgrade Workflow</a></li><li><a href=#video-version-of-this-post>Video Version of This Post</a></li></ul></li></ul></nav></div><p><strong>Prefer video? <a href=#video-version-of-this-post>This video</a> covers what’s in this
|
|
post with a little bit more detail.</strong></p><p>Checklists can be used for more than just database upgrades of course but I’m a
|
|
big fan of using practical examples to demonstrate something and I recently
|
|
performed a checklist driven database upgrade.</p><p>There’s a reason airline pilots use checklists even if they have 40+ years of
|
|
experience and have taken off and landed thousands of times. Having the
|
|
workflow in front of your face lets you focus on executing the steps instead of
|
|
trying to think about the workflow while executing the steps at the same time.</p><p>For one of my clients we were preparing a major database version upgrade (MySQL
|
|
5.7 to 8.0) and we wanted to minimize downtime and what could go wrong.</p><p>Needless to say a checklist was invaluable.</p><p>It let us do all of the hard work of defining the workflow while there’s no
|
|
pressure well before the time when the upgrade was happening. We also got a
|
|
chance to test the workflow in a test environment before applying it to the
|
|
live database.</p><p><a name=#database-upgrade-workflow></a></p><h3 id=database-upgrade-workflow><a href=#database-upgrade-workflow style=position:absolute;left:-12px>#</a>
|
|
Database Upgrade Workflow</h3><p>It was on AWS using RDS which is AWS’ managed MySQL / Postgres service but
|
|
similar steps could be used for other database upgrades too. After all of the
|
|
steps are listed out we’ll go over the details and my thought process of each
|
|
one in the next section.</p><p><em>None of the links below go anywhere. I added them here because in reality
|
|
these led to the exact resource being acted on in a step. This helps avoid
|
|
human error and needing to click around and find things on the fly. It’s a
|
|
minor but useful detail in my opinion.</em></p><ol><li>Ensure you can connect to the live database before doing anything</li><li>Confirm the SQL user has all necessary permissions to run any queries afterwards</li><li>Create <a href=#>maintenance page message</a> for example.com</li><li><a href=#>Enable maintenance page</a> at the firewall level</li><li><a href=#>Verify maintenance page</a> is being served for example.com</li><li>Create manual <a href=#>database snapshot</a> as a backup:<ul><li>Name it <code>example-pre-mysql8-upgrade-2023-05-08</code></li></ul></li><li><a href=#>Modify</a> the live <code>example</code> database:<ul><li>Set the DB engine to MySQL <code>8.0.32</code></li><li>Set the parameter group to <code>custom-mysql80-v1</code></li></ul></li><li>Wait until RDS is updated while <a href=#>looking at the status</a><ul><li>It may take 10-20 minutes</li></ul></li><li>Connect to the live database and run SQL queries to verify it works:<ul><li><code>SELECT ...</code> and expect <code>...</code></li><li><code>SELECT ...</code> and expect <code>...</code></li><li><code>SELECT ...</code> and expect <code>...</code></li></ul></li><li>Disable <a href=#>maintenance page</a></li><li>Verify <a href=#>example.com</a> is loading successfully</li><li>Create a calendar reminder to delete the <a href=#>manual snapshot</a> in 30 days</li><li>Notify the business that the site is back up and operational</li></ol><h4>Explanation</h4><p>Here’s my thought process for defining the checklist the way it was:</p><p><strong>Steps 1 and 2</strong> are critically important because you could get to step 9 and
|
|
then try to run a query only to find out that the DB user you’re connecting
|
|
with doesn’t have permissions.</p><p>For example, our “app” user can’t create or drop tables but our “migrate” user
|
|
can.</p><p>This happened in our test environment and it caused a blocker to obtain the
|
|
correct permissions because the person performing the upgrade (which was me in
|
|
this case) didn’t have access to modify SQL user permissions.</p><p>Step 1 is also nice because it’s a good idea to ensure your system is in
|
|
a working state before you modify it.</p><p><strong>Steps 3, 4 and 5</strong> are a handy way to toggle maintenance mode without needing
|
|
to perform a code change. It’s a firewall rule we can turn on and off on demand
|
|
for 1 or more domains.</p><p><strong>Step 6</strong> is an insurance policy in case something goes wrong we have the
|
|
latest version of our DB to ensure no data loss. That’s why we had to turn
|
|
maintenance mode on first to ensure no new data is being written to the DB.</p><p>Technically this isn’t flawless because in between steps 6 and 7 a background
|
|
job could modify the state of our database even with no new traffic being sent
|
|
to the site. For example any scheduled work that may happen at X time in the
|
|
future.</p><p>We have pretty robust logging and observability in place to know when this
|
|
happens along with a means to reconcile everything as needed if something went
|
|
wrong and we needed to use the snapshot.</p><p>Again, this only becomes an issue if we need to use the snapshot. If a
|
|
background job runs while the database is down during the upgrade itself that’s
|
|
ok, the job will fail and will be auto-retried. Eventually the DB will come
|
|
back up and it’ll work.</p><p><strong>Step 7</strong> is mainly clicking buttons in a web UI. That custom parameter group
|
|
was created ahead of time in preparation for the upgrade.</p><p>Ideally all of these “click buttons in web UI” steps could be put into code
|
|
using Terraform so they all become independent pull requests that can be
|
|
reviewed but this client has a 10+ year old app and not all of their
|
|
infrastructure has been ported into Terraform yet.</p><p><strong>Step 8</strong> is a waiting game. The AWS config tab for your RDS instance will
|
|
show the status, such as if it’s applying the new parameter group or if it’s in
|
|
the process of rebooting. The time range is pretty sporadic, I noticed upwards
|
|
of a 2x difference in total time based on nothing we could control.</p><p><strong>Step 9</strong> are a few sanity check queries to ensure things updated to their
|
|
correct version and certain tables are cleared based on what the dev team
|
|
suggested.</p><p><strong>Steps 10 and 11</strong> allow the site to be live again within a few seconds of
|
|
enabling it. Automated smoke tests would be a way to automate step 11 a bit but
|
|
for now a human went to a couple of pages to make sure they worked.</p><p><strong>Step 12</strong> is important because a snapshot costs money to store. We wanted to
|
|
keep it as a snapshot to quickly restore it as needed. We could have done a SQL
|
|
dump and stored it on S3 but realistically spending a couple of bucks to keep
|
|
the snapshot around is worth it.</p><p>If something goes wrong it lets the developers quickly spin up a test DB based
|
|
off the snapshot to help figure out what’s happening and address the situation.</p><p><strong>Step 13</strong> is always nice to let stakeholders know how things went. In our
|
|
case everything went flawlessly and we had around 15 minutes of expected
|
|
downtime which was done during a time where our traffic usually dips. The
|
|
preparation work paid off.</p><hr><p>You’ll notice there’s no steps or workflow for if something goes wrong. That’s
|
|
because I both wrote and executed the steps. At this point I was ok with
|
|
letting my overall general experience guide troubleshooting anything that might
|
|
go wrong.</p><p>If I were writing these steps for someone else, that would be different. I’d
|
|
add more details.</p><p>I also executed them in a test environment multiple times to work out the
|
|
kinks. By the time it was executed in production I was feeling really good
|
|
about it.</p><p>We did have a disaster recovery plan though. It wasn’t a formal document but we
|
|
thought about what we could do if certain things went wrong. Realistically
|
|
steps 7 and 8 are the riskiest ones and it’s technically out of our control.
|
|
Either RDS will successfully do it or not.</p><h4>While it’s Happening</h4><p>While executing the steps I created screenshots and quickly jotted down how
|
|
long each step took. I really like this idea because once the whole process is
|
|
finished you can go back and make a little post with a timeline that mentions
|
|
anything interesting that may have happened along the way.</p><p>This really helps your future self or someone else understand the process in
|
|
case they need to do a similar thing in the future. When performing a scary
|
|
upgrade, having the extra context and details is well worth the 15 minutes
|
|
writing up a summary when it happened.</p><p><a name=#video-version-of-this-post></a></p><h3 id=video-version-of-this-post><a href=#video-version-of-this-post style=position:absolute;left:-12px>#</a>
|
|
Video Version of This Post</h3><div style=position:relative;padding-bottom:56.25%;height:0;overflow:hidden><iframe allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share; fullscreen" loading=eager referrerpolicy=strict-origin-when-cross-origin src="https://www.youtube.com/embed/hOp7uog7_Xw?autoplay=0&controls=1&end=0&loop=0&mute=0&start=0" style=position:absolute;top:0;left:0;width:100%;height:100%;border:0 title="YouTube video"></iframe></div><p></p><h4>Timestamps</h4><ul><li>0:44 – Checklists are valuable</li><li>2:09 – Going over the database upgrade checklist</li><li>3:26 – Ensure you can connect to your DB</li><li>4:27 – Turning on maintenance mode for the site</li><li>5:55 – Create a manual snapshot as a backup and perform the upgrade</li><li>10:49 – Confirm it works with a few SQL queries</li><li>11:30 – Turning off maintenance mode for the site</li><li>12:10 – Create a calendar reminder to delete the snapshot backup</li><li>13:53 – Notifying the business that everything went well</li><li>14:24 – What if something went wrong?</li><li>16:30 – Making a timeline of interesting details afterwards</li><li>17:39 – What was the last thing you made a checklist for?</li></ul><p><strong>What was the last thing you made a checklist for? Let me know below.</strong></p></article><div class="cta-header-with-optin cta-header-with-optin-dark text-center"><h4>Never Miss a Tip, Trick or Tutorial</h4><form id=signup method=post class=form-horizontal><input type=hidden name=referrer value=https://nickjanetakis.com/blog/creating-checklists-for-high-stakes-changes>
|
|
<input type=hidden class=gmt name=gmt value=-4>
|
|
<input type=hidden name=list value=Hndxczl9v0YCEzbZ3VuVkg>
|
|
<input type=hidden name=subform value=yes><div class="form-group form-full-name" aria-hidden=true><div class="col-md-6 col-md-offset-3"><label class=sr-only for=full-name>Do not fill this field out as a human</label>
|
|
<input type=text id=full-name name=hp class=form-control placeholder="Full Name" tabindex=-1 autocomplete=off></div></div><div class=form-group><div class="col-md-6 col-md-offset-3"><label class=sr-only for=email>Email Address</label><div class=input-group><input type=email id=email name=email class=form-control placeholder="Email Address"><div class=input-group-btn><button type=submit class="btn btn-warning">
|
|
<strong>Get Updates</strong></button></div></div></div></div></form></div><p class="text-muted small half-margin-top text-center">Like you, I'm super protective of my inbox,
|
|
so don't worry about getting spammed. You can expect a few emails per year (at most), and
|
|
you can 1-click unsubscribe at any time. <a href=/newsletter>See what else you'll get</a> too.</p><hr><div class=social-share-btns-container><div class=social-share-btns><a class="share-btn share-btn-twitter" rel=noopener href="https://x.com/intent/tweet?url=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&text=Creating%20Checklists%20for%20High%20Stakes%20Changes%20by%20%40nickjanetakis"><i class="fa fa-twitter"></i>
|
|
Share
|
|
</a><a class="share-btn share-btn-facebook" rel=noopener href="https://www.facebook.com/sharer/sharer.php?u=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&title=Creating%20Checklists%20for%20High%20Stakes%20Changes"><i class="fa fa-facebook"></i>
|
|
Share
|
|
</a><a class="share-btn share-btn-linkedin" rel=noopener href="http://www.linkedin.com/shareArticle?mini=true&url=https%3a%2f%2fnickjanetakis.com%2fblog%2fcreating-checklists-for-high-stakes-changes&title=Creating%20Checklists%20for%20High%20Stakes%20Changes"><i class="fa fa-linkedin"></i>
|
|
Share</a></div></div><hr><h2>Comments</h2><div id=disqus_thread data-permalink=https://nickjanetakis.com/blog/creating-checklists-for-high-stakes-changes data-relpermalink=/blog/creating-checklists-for-high-stakes-changes></div><script>var disqus_config=function(){var e=document.querySelector("#disqus_thread");this.page.url=e.dataset.permalink,this.page.identifier=e.dataset.relpermalink};(function(){var e=document,t=e.createElement("script");t.src="//nickjj.disqus.com/embed.js",t.setAttribute("data-timestamp",+new Date),(e.head||e.body).appendChild(t)})()</script><noscript>Please enable JavaScript to view the <a href=https://disqus.com/?ref_noscript rel=nofollow>comments powered by Disqus.</a></noscript><script id=dsq-count-scr src=//nickjj.disqus.com/count.js async></script><hr><footer><p>© 2026 Nick Janetakis</p><a target=_blank class="small twitter-follow-button" href=https://x.com/nickjanetakis><span class=text-primary>Follow @nickjanetakis</span></a><p class="small tiny-margin-top"><a target=_blank href=https://github.com/nickjj><span class="fa fa-github text-black"></span> GitHub
|
|
</a>|
|
|
<a target=_blank href=https://www.youtube.com/c/nickjanetakis><span class="fa fa-youtube text-black"></span> YouTube</a></p></footer></div></div><script type=text/javascript src=/assets/js/app.9b2ef33b81dccfc0bf4a638a2b2041902827deb5ad23846d48c1a506bc6b21c6.js></script></body></html> |