Files
nexus/sreweekly/articles/179/04-it-s-dead-jim-how-we-write-an-incident-postmortem.html
2026-09-12 17:23:01 +08:00

339 lines
44 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><!-- Last Published: Wed Jul 02 2025 01:43:30 GMT+0000 (Coordinated Universal Time) --><html data-wf-domain="webflow.hostedgraphite.com" data-wf-page="5ce3e821b5d6678199f31df7" data-wf-site="5a57aa76d1fa2300015ca244" data-wf-collection="5ce3e821b5d6674c4bf31e14" data-wf-item-slug="its-dead-jim-how-we-write-an-incident-postmortem"><head><meta charset="utf-8"/><link href="https://cdn.prod.website-files.com" rel="preconnect" crossorigin="anonymous"/><title>&quot;It&#x27;s dead, Jim&quot;: How we write an incident postmortem</title><meta content="How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper." name="description"/><meta content="&quot;It&#x27;s dead, Jim&quot;: How we write an incident postmortem" property="og:title"/><meta content="How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper." property="og:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png" property="og:image"/><meta content="&quot;It&#x27;s dead, Jim&quot;: How we write an incident postmortem" name="twitter:title"/><meta content="How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper." name="twitter:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png" name="twitter:image"/><meta property="og:type" content="website"/><meta content="summary_large_image" name="twitter:card"/><meta content="width=device-width, initial-scale=1" name="viewport"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/css/hosted-graphite-blog.webflow.shared.f88b646c4.min.css" rel="stylesheet" type="text/css"/><link href="https://fonts.googleapis.com" rel="preconnect"/><link href="https://fonts.gstatic.com" rel="preconnect" crossorigin="anonymous"/><script src="https://ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js" type="text/javascript"></script><script type="text/javascript">WebFont.load({ google: { families: ["Open Sans:300,300italic,400,400italic,600,600italic,700,700italic,800,800italic","Inconsolata:400,700","Montserrat:100,100italic,200,200italic,300,300italic,400,400italic,500,500italic,600,600italic,700,700italic,800,800italic,900,900italic","Lato:100,100italic,300,300italic,400,400italic,700,700italic,900,900italic","Droid Sans:400,700","Vollkorn:400,400italic,700,700italic","Ubuntu:300,300italic,400,400italic,500,500italic,700,700italic","Lora:regular,italic,700","Oxygen:300,regular,700","Source Sans Pro:200,200italic,300,300italic,regular,600,600italic,700,700italic,900,900italic"] }});</script><script type="text/javascript">!function(o,c){var n=c.documentElement,t=" w-mod-";n.className+=t+"js",("ontouchstart"in o||o.DocumentTouch&&c instanceof DocumentTouch)&&(n.className+=t+"touch")}(window,document);</script><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f272bbb5a83b7b3239ad6_32.png" rel="shortcut icon" type="image/x-icon"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f27323f9c40d523c9bfcc_256.png" rel="apple-touch-icon"/><script src="https://cdn.jsdelivr.net/npm/code-prettify@0.1.0/src/prettify.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script type="text/javascript">
var embeddify = function(){
$("h6").each(function(index){
var h6 = $(this);
var url = h6.text();
console.log("Embeddifying " + url);
h6.hide();
var embed_message = $('<pre>Embedding <a href="'+url+'" target="_blank">some code</a>, one moment...</pre>');
embed_message.insertAfter(h6);
$.ajax(url, {
success: function(content){
$('<?prettify linenums=true?><pre class="prettyprint">'+content+'</pre>').insertAfter(h6);
embed_message.hide();
PR.prettyPrint();
},
error: function(jqXHR, textStatus, errorThrown){
embed_message.html('Tried to fetch <a href="'+url+'" target="_blank">some code from GitHub</a>, but it failed with an error: "'+errorThrown+'"');
}
});
});
};
var webflow_removed = false;
var attempts_remaining = 10;
function dewebflow() {
/* Temporarily remove this badge while we're sorting out
serving this from the right domains so the hosting provider
will remove the badge properly. */
var badges = $("a.w-webflow-badge");
if(badges.length == 1)
{
$("a.w-webflow-badge").remove();
webflow_removed = true;
} else {
attempts_remaining--;
}
if(!webflow_removed || attempts_remaining > 0)
setTimeout(dewebflow, 500);
}
$(document).ready(embeddify);
$(document).ready(dewebflow);
</script>
<style type"text/css">
/* Turn on line numbering for every line of prettyprinted code, instead of every fifth line. */
li.L0, li.L1, li.L2, li.L3,
li.L5, li.L6, li.L7, li.L8 {
list-style-type: decimal !important;
}
</style>
<!-- Google Tag Manager -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer','GTM-NT79GSP');</script>
<!-- End Google Tag Manager --></head><body class="body-9"><div data-collapse="medium" data-animation="default" data-duration="200" data-easing="ease" data-easing2="ease" role="banner" class="navbar-9 w-nav"><div class="section-15"><div class="container-12 w-container"><nav role="navigation" class="nav-menu-5 w-clearfix w-nav-menu"><a href="https://www.hostedgraphite.com/accounts/signup/" class="hg-nav-link right marketing get-started-ppc-button w-nav-link">Get started free</a><a href="https://www.hostedgraphite.com/accounts/login/" class="hg-nav-link right w-nav-link">Login</a><a href="https://www.hostedgraphite.com/enterprise" class="hg-nav-link w-nav-link">Enterprise</a><a href="https://www.hostedgraphite.com/pricing" class="hg-nav-link w-nav-link">Pricing</a><a href="https://www.hostedgraphite.com/docs/" class="hg-nav-link w-nav-link">Docs</a><a href="https://www.hostedgraphite.com/customers" class="hg-nav-link w-hidden-medium w-hidden-small w-hidden-tiny w-nav-link">Customers</a><a href="https://www.hostedgraphite.com/product" class="hg-nav-link w-nav-link">Features</a></nav><div class="menu-button-3 w-nav-button"><div class="w-icon-nav-menu"></div></div></div></div><div class="div-block-15"><a href="https://www.hostedgraphite.com" class="brand-4 w-nav-brand"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg" width="1088.5" alt="" class="image-52"/></a></div><div class="section-16"></div></div><div data-w-expand="category" style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="hero-blog"></div><div class="main-section"><div class="w-container"><div class="section-heading"><h1 class="blog-post-title">&quot;It&#x27;s dead, Jim&quot;: How we write an incident postmortem</h1><div class="blog-date">July 31, 2019</div><a data-w-expand="category" href="/blog-categories/engineering" class="blog-category">Engineering</a><div class="full-divide"></div><div class="w-embed w-script"><script>
document.addEventListener("DOMContentLoaded", function() {
const tocContainer = document.createElement('div');
tocContainer.id = 'toc-container';
tocContainer.innerHTML = '<h2>Table of Contents</h2><ul id="toc"></ul>';
const sectionHeading = document.querySelector('.section-heading');
sectionHeading.insertAdjacentElement('afterend', tocContainer);
const toc = document.getElementById('toc');
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
headers.forEach((header) => {
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
header.id = id;
const li = document.createElement('li');
const a = document.createElement('a');
a.href = `#${id}`;
a.textContent = header.textContent;
li.appendChild(a);
toc.appendChild(li);
});
document.querySelectorAll('#toc a').forEach(anchor => {
anchor.addEventListener('click', function(e) {
e.preventDefault();
document.querySelector(this.getAttribute('href')).scrollIntoView({
behavior: 'smooth'
});
history.pushState(null, null, this.getAttribute('href'));
});
});
});
</script>
<style>
#toc-container {
text-align: start;
width: 85%;
margin: 0 auto 3em;
}
#toc-container h2 {
margin-top: 0;
}
#toc-container ul {
padding-left: 1em;
}
#toc-container li {
margin-bottom: 0.5em;
}
#toc li a {
text-decoration: none;
color: #2e2e2e;
}
#toc li a:hover {
text-decoration: underline;
}
</style></div></div><div class="w-condition-invisible w-embed"></div><div class="blog-content"><div class="blog-post w-richtext"><p>No matter how hard we try to offer an uninterrupted service, outages are inevitable. Fixing the underlying issue and <a href="https://www.hostedgraphite.com/blog/how-to-write-a-status-page-update">notifying customers</a> is crucial, but it shouldn’t end there. There must be a process in place to learn from what happened and make sure it doesn’t happen again.</p><p>In the penultimate part of our SRE process series, we look at how to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper.<br/></p><h2>What is a postmortem?</h2><p>A postmortem is the written record of an incident, including its impact, actions taken to mitigate it, and lessons learned from it (including followup tasks). While the focus of our incident management is usually in mitigating a currently ongoing incident, the goal of the postmortem is to look forward, and try to make sure we have learned as much as we can from a given incident, so we can be in a better position than we were before to avoid similar issues from reoccurring in the future. If we don&#x27;t do this we&#x27;ll be fighting fires every day, and that&#x27;s no fun.</p><p>In other words, a postmortem is the process by which we learn from failure, and a way to document and communicate those lessons. The more we fail, the more learning opportunities we have.</p><h2>Why are postmortems important?</h2><p>There are several reasons postmortems are an incredibly important tool:</p><ul role="list"><li>It allows us to document the incident, ensuring that it won&#x27;t be forgotten. A well-documented incident is invaluable because it includes not only a description of what happened but of what actions we took and the things we believed to be true at the time, which can help inform our actions during future incidents.</li><li>They are the most effective mechanism we can use to drive improvement in our infrastructure. Nothing like seeing our services and processes fail in new and interesting ways to realise what areas need improvement.</li><li>It helps shift the focus from the immediate <strong>now</strong> (&quot;we need to mitigate the impact from this incident now&quot;) to the future (&quot;what can we do to improve our systems so this incident doesn&#x27;t reoccur?&quot;).</li><li>When they&#x27;re posted publicly, it lets our users know that we take every outage seriously, and that we&#x27;re doing all we can to learn from them and prevent any future disruptions to the service we provide.</li></ul><h2>What&#x27;s the goal of a postmortem?</h2><p>The number one goal of a postmortem is to learn things from it. It&#x27;s not a great sign when you sit down to write a postmortem and you already know everything you&#x27;re going to say. If we&#x27;re not learning anything, we aren&#x27;t digging deep enough.</p><p>The final postmortem document is just a (small) part of our postmortem process, and its value lies in sharing (both with the rest of the team and the outside world) the important lessons we have learned, so the goal of our postmortem process is not just to produce a document. This document is merely the conduit by which we share what we learned on our journey of discovery. In a way, you could say that the real postmortem was the friends we made along the way.</p><h2>Why do we share our postmortems?</h2><p>We believe in being open with our customers, and we take this very seriously with our <a href="https://www.hostedgraphite.com/blog/how-to-write-a-status-page-update">customer communication during incidents</a>, so publishing our lessons learned after an incident is just an extension of this. Our customers deserve to know why their service wasn&#x27;t working the way they expect it to work and that when we tell them we&#x27;ll do better in the future we&#x27;re not just saying it, and we have actual steps we&#x27;ll take to ensure that&#x27;s the case.</p><h2>Postmortem process</h2><p>So we just had an incident. It probably required posting something on our <a href="https://status.hostedgraphite.com/">status page</a>, and now status page is giving you the option to write and publish a postmortem for this incident. This is where the fun begins.</p><h2>Remember, a postmortem is not just a document</h2><p>We already said that the goal of a postmortem process is not just to produce a document, but to learn from failure as much as we can. This means that part of this process is going to involve asking some hard questions to try to extract as much <a href="https://www.youtube.com/watch?v=quTn9pL39Cw">learning juice</a> as we can from failure. This means that if we feel we don&#x27;t have much to say on a given postmortem it could very well be because we haven&#x27;t dug too deeply into this particular incident and everything surrounding it.</p><p>For example, we shouldn&#x27;t be satisfied with identifying what triggered an incident (after all, <a href="https://www.kitchensoap.com/2012/02/10/each-necessary-but-only-jointly-sufficient/">there is no root cause</a>), but should use the opportunity to investigate all the contributing factors that made it possible, and/or how our automation might have been able to prevent this from ever happening. The lessons we learn from an incident only stop coming when we stop digging, so an incident with no lessons learned only means we didn&#x27;t look hard enough.</p><h2>When should I write a postmortem?</h2><p>Postmortems are such a good learning opportunity that we should take every chance we get to write one, but the decision on writing one or not usually falls on the incident commander (normally the on-call engineer at the time). If we aren&#x27;t sure if a given incident &quot;deserves&quot; a postmortem, <strong>it&#x27;s never a bad choice to write one anyway</strong>, it&#x27;s best to err on the side of oversharing than to give the impression that we don&#x27;t care enough about communicating about our incidents, and we should always be happy to have another learning opportunity.</p><p>If you think an incident is &quot;too common&quot; to get its own postmortem that&#x27;s a good indicator that there&#x27;s a deeper issue that we need to address, and an excellent opportunity to apply our postmortem process to it. Sometimes a single instance of an incident can&#x27;t give you enough information to get any meaningful lessons out of it, but when looking at a group of seemingly related incidents as an aggregate they might start to paint a clearer picture.</p><p>If we know that we&#x27;ll want to write a postmortem before officially resolving the incident on our status page, it&#x27;s always a good idea to tell our customers to expect a postmortem. A postmortem doesn&#x27;t need to go out on the same day the incident happened, and there&#x27;s certainly no expectation of staying until late or over the weekend writing one. Having a postmortem ready on the next business day after an incident is a good goal, but in some cases (such as particularly complex incidents, or times where we&#x27;re still very busy dealing with the fallout) this could be delayed a bit more. Ideally, it should never take more than a week after the incident is resolved for the postmortem to be published.</p><p>It&#x27;s also worth noting that not all postmortems need to be published on our status page or be tied to an actual incident. Sometimes we&#x27;ll want to write a postmortem around near-incidents or incidents that didn&#x27;t have enough of a visible impact to warrant updating our customers. A postmortem doesn&#x27;t need to be published externally to be useful.</p><h2>Who should be writing this postmortem?</h2><p>It&#x27;s usually up to the on-call engineer to write a postmortem for any incidents that happened during <a href="https://pics.me.me/when-your-shift-is-over-and-your-manager-asks-you-4760512.png">their watch</a>, but as with many other things regarding on-call, this too can be delegated. The on-call engineer is still responsible for ensuring that we produced a postmortem and that it&#x27;s shared both internally and publicly, but they don&#x27;t necessarily need to write it themselves.</p><p>That said, just because one person is leading this process doesn&#x27;t mean a postmortem is a one-person job. A postmortem is a team effort and you&#x27;ll want input from everybody that was involved in the incident (and others that weren&#x27;t). We all have a different perspective and a different mental model of what our systems look like, so only by combining them all, you&#x27;ll get closer to the full picture of what really happened during an incident.</p><h2>Is this going to be a finger-pointing exercise?</h2><p><a href="https://www.youtube.com/watch?v=GGn25URIss8">No</a>. We could fill pages talking about blameless postmortems and how important we are, but the main takeaway is that we all make mistakes, and we&#x27;re not here to point at those and say &quot;our problem is that someone made a mistake, we&#x27;ll try making zero mistakes next time&quot;. What we want is to learn why our processes allowed for that mistake to happen, to understand if the person that made a mistake was operating under wrong assumptions (and how our people can have the necessary information to make better decisions) or even why they were doing what they were doing in the first place (instead of that process being fully automated).</p><p>Nobody gets blamed when something goes wrong, but the more we share about these experiences, the more we&#x27;ll learn about them. Despite not assigning blame, we can (and should!) explicitly identify times where a mistake was made.</p><p>It&#x27;s important to call out mistakes, but our focus should be on the mistake itself and what can be learned from it, as opposed to the person making the mistake. That person becomes our leading expert on that particular mistake, so we&#x27;ll want to learn from everything they have to teach us.</p><h2>So where do I start?</h2><p>The first (and most important) steps of the postmortem process don&#x27;t require us to write a single word. Before we can put everything we&#x27;ve learned in a document we have to truly understand the incident.</p><ol role="list"><li>Compile a timeline of the incident. It&#x27;s really useful to see what actions we took and when we took them, and the things we thought to be true at any point during the incident.</li><li>Ask yourself (and others) <strong>a lot of questions</strong>. We know <a href="https://www.kitchensoap.com/2012/02/10/each-necessary-but-only-jointly-sufficient/">there is no (single) root cause</a>, and that the story of an incident is composed of <a href="https://www.oreilly.com/ideas/the-infinite-hows">infinite hows</a>, which means that a postmortem will only be useful if we continue digging and challenging any assumptions we have about or systems. Some examples of useful questions would include (but are certainly not limited to):</li></ol><ul role="list"><li>How did this failure go unnoticed for XX minutes? Do we not have alerts that cover this failure scenario? Did they work as expected?</li><li>Even in cases when we still don&#x27;t know why something happened and remains a mystery, what kind of instrumentation/diagnostics do we think we&#x27;d need to be able to identify it the next time it happens?</li><li>Did we accurately assess the impact originally? If we didn&#x27;t, how can we make sure we do it better the next time?</li><li>Could the incident have been worse but maybe we got lucky somehow? What could happen if the next time we don&#x27;t have that kind of luck?</li><li>Did we get unlucky and an incident that shouldn&#x27;t have been a major issue somehow became one? Then we need to dig into what were the contributing factors to that, since &quot;have better luck next time&quot; is not the best strategy.</li><li>Was the incident caused or made worse by something we did? What led us to believe that was the right course of action? Could our systems/tooling have prevented us from taking that action or mitigate its impact? Remember, our postmortems are blameless so this is not a finger-pointing exercise, but we need to be able to identify these instances so we can look at all the contributing factors.</li><li>Did we, at some point, make the wrong call? Did we have invalid/incomplete information at the time? Maybe our documentation was the issue?</li><li>What kind of information would we need to do better next time?</li><li>What was each of us thinking during the incident? How did we feel? Did we feel we had the right information/context at all times? The people involved in the incident are also a part of the system we&#x27;re trying to learn about, and as such it&#x27;s important not to overlook them.</li></ul><p>While working through the timeline and asking questions you should be making a note of everything that makes you think &quot;hmmm maybe this could have gone better if only we had X&quot;, as those will end up becoming our follow-up actions for this postmortem.</p><p>Next up, we’ll take a deeper look at the structure of a postmortem–section by section–with a helpful template, writing tips, and some pointers to keep in mind.</p><h2>Related reading</h2><p>There are too many great resources out there to list, but the following should be considered required reading (or watching!) on the topic:</p><p><a href="https://landing.google.com/sre/book/chapters/postmortem-culture.html">Chapter 15 of the SRE book</a></p><p><a href="https://www.oreilly.com/ideas/the-infinite-hows">&quot;The infinite hows&quot; - John Allspaw</a></p><p><a href="https://www.youtube.com/watch?v=8DtzmV1jiyQ">Incidents as we Imagine Them Versus How They Actually Are - John Allspaw (video)</a></p><p><a href="https://web.mit.edu/2.75/resources/random/How%20Complex%20Systems%20Fail.pdf">How complex systems fail - Richard Cook</a></p><p><a href="https://www.adaptivecapacitylabs.com/blog/2018/10/08/the-multiple-audiences-and-purposes-of-post-incident-reviews/">The Multiple Audiences and Purposes of Post-Incident Reviews</a></p><p><a href="https://www.adaptivecapacitylabs.com/blog/2019/06/17/some-observations-on-the-messy-realities-of-incident-reviews/">Some Observations On the Messy Realities of Incident Reviews</a></p><p><a href="https://www.adaptivecapacitylabs.com/blog/2019/03/03/hindsight-and-sacrifice-decisions/">Hindsight and sacrifice decisions</a></p><p><br/></p></div><div class="sticky-toc-container"><div class="cta-sidebar"><div class="text-block-31"><strong class="bold-text-15">Try Hosted Graphite now!</strong></div><p class="paragraph-12">Get Hosted Graphite free for 14 days. No credit card required.</p><a href="https://www.hostedgraphite.com/accounts/signup/?from=heroku_blogpost" class="cta-side-button w-button">Get Started</a></div><div class="toc w-embed w-script"><script>
document.addEventListener("DOMContentLoaded", function() {
function createSideToc(containerId) {
const container = document.querySelector(containerId);
// Create and append the title <h2>
const tocTitle = document.createElement('h2');
tocTitle.textContent = 'Table of Contents';
container.appendChild(tocTitle);
const toc = document.createElement('ul');
toc.className = 'toc';
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
const ids = new Set(); // To ensure unique IDs
headers.forEach((header) => {
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
header.id = id;
if (!ids.has(id)) {
ids.add(id);
const li = document.createElement('li');
const a = document.createElement('a');
a.href = `#${id}`;
a.textContent = header.textContent;
li.appendChild(a);
toc.appendChild(li);
}
});
container.appendChild(toc);
// Smooth scroll for the new TOC
toc.querySelectorAll('a').forEach(anchor => {
anchor.addEventListener('click', function(e) {
e.preventDefault();
document.querySelector(this.getAttribute('href')).scrollIntoView({
behavior: 'smooth'
});
history.pushState(null, null, this.getAttribute('href'));
});
});
}
// Create the sticky TOC in the sidebar
createSideToc('.sticky-toc-container');
});
</script>
<style>
/* Specific Styling for Sticky TOC */
.sticky-toc-container {
position: sticky;
top: 110px;
width: 340px;
height: 100%;
overflow-y: auto;
padding: 15px;
border-radius: 8px;
font-family: Arial, sans-serif;
font-size: 14px;
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
text-align: left;
}
.sticky-toc-container h2 {
margin-top: 0;
font-size: 18px;
color: #2e2e2e;
}
.sticky-toc-container ul {
list-style-type: none;
padding-left: 0;
margin: 0;
}
.sticky-toc-container li {
margin-bottom: 0.6em;
}
.sticky-toc-container a {
text-decoration: none;
color: #8e8b8b;
}
.sticky-toc-container a:hover {
text-decoration: underline;
}
/* Make sure the TOC has a scroll bar when needed */
.toc {
max-height: 35vh;
overflow-y: auto;
}
/* Media Query to Hide sticky-toc-container on smaller screens */
@media (max-width: 1215px) {
.sticky-toc-container {
display: none;
}
}
</style></div></div></div><div class="w-condition-invisible w-embed w-script"><script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js"></script>
<pre class="prettyprint">
</pre></div><div class="div-block-2"><div class="full-divide"></div><div class="author-wrapper"><a href="/blog-authors/fran-garcia" data-w-expand="authors" class="author-name">Fran Garcia</a><div class="smallest-divider"></div><div data-w-expand="authors" class="author-bio w-richtext"><p>SRE at Hosted Graphite.</p></div><div><a href="https://twitter.com/hostedgraphite" data-w-expand="authors" class="social-link w-inline-block"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a57aa7ad1fa2300015ca29a_social-18.svg" width="68" alt="" class="image-2"/></a><a data-w-expand="authors" href="#" class="social-link w-inline-block"></a><a href="#" data-w-expand="authors" class="social-link w-inline-block"></a></div></div></div></div></div><div class="main-section gray"><div class="w-container"><div class="section-heading"><h2 class="heading-6">Related Posts</h2><div class="med-divider"></div></div><div class="w-dyn-list"><div role="list" class="w-clearfix w-dyn-items w-row"><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/hosted-graphite-isnt-graphite" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5dd37e9fe192cd2ea5b8af68_Screen%20Shot%202019-11-19%20at%202.14.02%20PM.jpg&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">HostedGraphite</div><div class="preview-text">Hosted Graphite improves upon standard Graphite. Take a look at how we do this and how Hosted Graphite gives your company better functionality. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Shevaun Frazier</div><div class="thumbnail-date">Nov 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/new-developer-onboarding" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5bfea01e7ffa1b30f88f19c0_Blog-background-organge.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">New Developer Onboarding</div><div class="preview-text">Discussions about onboarding tend to revolve around new hires, for obvious reasons, but the process is important for everyone: while the new developer learns the most important aspects of team and company culture, the team has an opportunity to learn new ideas from a fresh pair of eyes. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Heather Wiencko</div><div class="thumbnail-date">Sep 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/pug-life-how-we-run-a-grafana-instance-for-each-user-with-docker" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5d643d90b1c59597a144484a_pug-header.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">PUG Life: How we run a Grafana instance for each user with Docker </div><div class="preview-text">The story of Hosted Grafana at Hosted Graphite, and how we run multiple instances of Grafana. Start monitoring with Hosted Graphite and use our Grafana dashboards directly in-platform. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciarán Finn</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/incident-postmortem-template" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Our incident postmortem template</div><div class="preview-text">Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Aug 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/surviving-on-call-tips-from-a-hosted-graphite-sre" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c48a4405b91aaf448087677_daveblog-header.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Surviving On-Call: Tips from a Hosted Graphite SRE</div><div class="preview-text">Get an insight into what Hosted Graphite&#x27;s SRE Dave Fennell has learned when being On-Call. Read the tips that he has to offer to make the On-Call experience a positive one.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Dave Fennell</div><div class="thumbnail-date">Jan 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/a-victim-of-its-own-popularity-scaling-our-cloudwatch-integration" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">A victim of its own popularity: Scaling our CloudWatch integration</div><div class="preview-text">Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Egan</div><div class="thumbnail-date">Dec 2018</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/deadlines-lies-and-videotape-the-tale-of-a-grpc-bug" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Deadlines lies and videotape: The tale of a gRPC bug</div><div class="preview-text">If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released).</div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Gaffney</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/status-page-updates-its-all-about-timing" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Status page updates: It’s all about timing</div><div class="preview-text">Check out part 2 of Hosted Graphite&#x27;s SRE process on how to handing status page updates. Learn more in detail on the responsibilities of communication, duration of updating the page, and who’s responsible for certain tasks.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Oct 2018</div></div></a></div></div></div></div></div><div class="main-section dark"><div class="container-13 w-container"><div class="section-heading"><h2 class="white">See why thousands of engineers trust Hosted Graphite with their monitoring</h2><div class="med-divider"></div></div><a href="https://www.hostedgraphite.com/accounts/signup/" class="get-started-ppc-button w-button">START A FREE TRIAL</a><div class="div-block-14 w-hidden-medium w-hidden-small w-hidden-tiny"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png" width="120" sizes="(max-width: 991px) 100vw, 120px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1600.png 1600w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png 2000w" alt="" class="image-5"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png" width="190" sizes="(max-width: 991px) 100vw, 190px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png 1280w" alt="" class="image-3"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1)-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png 620w" alt="" class="image-4"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png 800w" alt="" class="image-6"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd52f6b273e0001a3d86c_Tableau%20(1).svg" alt="" class="image-7"/></div></div><div class="footer-section"><div class="w-container"><div>© 2024 Metricfire Limited. <br/>All Rights Reserved.<br/><br/><br/>5940 S Rainbow Blvd Ste 400<br/>Las Vegas, NV 89118-2507<br/>United States</div></div></div></div><div class="w-embed"></div><div class="w-embed w-script"><script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Article",
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "https://www.hostedgraphite.com/blog/its-dead-jim-how-we-write-an-incident-postmortem"
},
"headline": "&quot;It&#39;s dead, Jim&quot;: How we write an incident postmortem",
"image": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png",
"thumbnailUrl": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png",
"author": {
"@type": "Person",
"name": "Fran Garcia",
"url": "fran-garcia"
},
"publisher": {
"@type": "Organization",
"name": "Metricfire Limited",
"logo": {
"@type": "ImageObject",
"url": "https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg"
}
},
"datePublished": "Jul 31, 2019",
"dateModified": "Sep 07, 2021",
"description": "How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper."
}
</script>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "SoftwareApplication",
"applicationCategory": "BusinessApplication",
"name": "Hosted Graphite",
"description": "Hosted Graphite offers powerful monitoring and alerting for various business applications, providing integrations with multiple data sources, a Grafana UI, and great customer support.",
"review": [
{
"@type": "Review",
"author": { "@type": "Person", "name": "Cecily G." },
"reviewBody": "This system is the tool every engineer needs to be successful. Easy to use and captures metrics that no other system has successfully been able to pull for us.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Christopher C." },
"reviewBody": "Great product suite, outstanding customer service! Running a distributed data pipeline is challenging to keep healthy, and Hosted Graphite has been an instrumental partner for monitoring system health and getting ahead of issues.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "UC" },
"reviewBody": "A great service with excellent support. Simple to use, inexpensive, and great support.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Brendan C." },
"reviewBody": "High-quality metrics hosting. Hosted Graphite enables us to instrument our products and processes efficiently. We can proactively monitor for issues and use dashboards to identify more minor performance problems before they become major ones.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Jacobson M." },
"reviewBody": "Perfect for remote device analytics. Hosted graphite makes it easy for me to keep track of my remote devices being used by my customers. I can't always be on site, so this allows easy tracking of device details.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "James C." },
"reviewBody": "The easiest way for DevOps to incorporate Kubernetes without having to suffer a paradigm shift away from using Graphite. Prometheus, while academically good, forces security worst practices which were unacceptable for us as a SOC2 compliant organization. Graphite is battle-proven and already integrated, so Hosted Graphite allowed us to do what we needed without drastic changes.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
"datePublished": "2021-03-19"
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Anne M." },
"reviewBody": "Offers great hosting service, Hosted Graphite is quite easy to use and highly secure. I like how they offer the trial version to evaluate if the solution fits your needs. In terms of setup, it's easy with minimal registration details and flexible pricing.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
"datePublished": "2019-02-20"
}
],
"operatingSystem": "All major operating systems",
"aggregateRating": {
"@type": "AggregateRating",
"reviewCount": 7,
"ratingValue": 4.85,
"bestRating": 5,
"worstRating": 1
},
"offers": {
"@type": "Offer",
"price": 19.00,
"priceCurrency": "USD",
"availability": "https://schema.org/InStock",
"url": "https://hostedgraphite.com",
"seller": {
"@type": "Organization",
"name": "Hosted Graphite"
}
}
}
</script></div><script src="https://d3e54v103j8qbb.cloudfront.net/js/jquery-3.5.1.min.dc5e7f18c8.js?site=5a57aa76d1fa2300015ca244" type="text/javascript" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.36b8fb49256177c8.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.83de43c5e9eda212.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.54e1a7f0.3d4248ede2db61fe.js" type="text/javascript"></script><!-- Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-NT79GSP"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) --><script>
img = document.querySelectorAll('.w-embed');
img.forEach(element => element.style.textAlign = 'center');
</script></body></html>