339 lines
41 KiB
HTML
339 lines
41 KiB
HTML
<!DOCTYPE html><!-- Last Published: Wed Jul 02 2025 01:43:30 GMT+0000 (Coordinated Universal Time) --><html data-wf-domain="webflow.hostedgraphite.com" data-wf-page="5ce3e821b5d6678199f31df7" data-wf-site="5a57aa76d1fa2300015ca244" data-wf-collection="5ce3e821b5d6674c4bf31e14" data-wf-item-slug="incident-postmortem-template"><head><meta charset="utf-8"/><link href="https://cdn.prod.website-files.com" rel="preconnect" crossorigin="anonymous"/><title>Our incident postmortem template</title><meta content="Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples." name="description"/><meta content="Our incident postmortem template" property="og:title"/><meta content="Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples." property="og:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png" property="og:image"/><meta content="Our incident postmortem template" name="twitter:title"/><meta content="Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples." name="twitter:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png" name="twitter:image"/><meta property="og:type" content="website"/><meta content="summary_large_image" name="twitter:card"/><meta content="width=device-width, initial-scale=1" name="viewport"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/css/hosted-graphite-blog.webflow.shared.f88b646c4.min.css" rel="stylesheet" type="text/css"/><link href="https://fonts.googleapis.com" rel="preconnect"/><link href="https://fonts.gstatic.com" rel="preconnect" crossorigin="anonymous"/><script src="https://ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js" type="text/javascript"></script><script type="text/javascript">WebFont.load({ google: { families: ["Open Sans:300,300italic,400,400italic,600,600italic,700,700italic,800,800italic","Inconsolata:400,700","Montserrat:100,100italic,200,200italic,300,300italic,400,400italic,500,500italic,600,600italic,700,700italic,800,800italic,900,900italic","Lato:100,100italic,300,300italic,400,400italic,700,700italic,900,900italic","Droid Sans:400,700","Vollkorn:400,400italic,700,700italic","Ubuntu:300,300italic,400,400italic,500,500italic,700,700italic","Lora:regular,italic,700","Oxygen:300,regular,700","Source Sans Pro:200,200italic,300,300italic,regular,600,600italic,700,700italic,900,900italic"] }});</script><script type="text/javascript">!function(o,c){var n=c.documentElement,t=" w-mod-";n.className+=t+"js",("ontouchstart"in o||o.DocumentTouch&&c instanceof DocumentTouch)&&(n.className+=t+"touch")}(window,document);</script><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f272bbb5a83b7b3239ad6_32.png" rel="shortcut icon" type="image/x-icon"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f27323f9c40d523c9bfcc_256.png" rel="apple-touch-icon"/><script src="https://cdn.jsdelivr.net/npm/code-prettify@0.1.0/src/prettify.min.js"></script>
|
||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
|
||
<script type="text/javascript">
|
||
var embeddify = function(){
|
||
$("h6").each(function(index){
|
||
var h6 = $(this);
|
||
var url = h6.text();
|
||
console.log("Embeddifying " + url);
|
||
h6.hide();
|
||
var embed_message = $('<pre>Embedding <a href="'+url+'" target="_blank">some code</a>, one moment...</pre>');
|
||
embed_message.insertAfter(h6);
|
||
$.ajax(url, {
|
||
success: function(content){
|
||
$('<?prettify linenums=true?><pre class="prettyprint">'+content+'</pre>').insertAfter(h6);
|
||
embed_message.hide();
|
||
PR.prettyPrint();
|
||
},
|
||
error: function(jqXHR, textStatus, errorThrown){
|
||
embed_message.html('Tried to fetch <a href="'+url+'" target="_blank">some code from GitHub</a>, but it failed with an error: "'+errorThrown+'"');
|
||
}
|
||
});
|
||
});
|
||
};
|
||
|
||
var webflow_removed = false;
|
||
var attempts_remaining = 10;
|
||
|
||
function dewebflow() {
|
||
/* Temporarily remove this badge while we're sorting out
|
||
serving this from the right domains so the hosting provider
|
||
will remove the badge properly. */
|
||
|
||
var badges = $("a.w-webflow-badge");
|
||
if(badges.length == 1)
|
||
{
|
||
$("a.w-webflow-badge").remove();
|
||
webflow_removed = true;
|
||
} else {
|
||
attempts_remaining--;
|
||
}
|
||
|
||
if(!webflow_removed || attempts_remaining > 0)
|
||
setTimeout(dewebflow, 500);
|
||
}
|
||
|
||
$(document).ready(embeddify);
|
||
$(document).ready(dewebflow);
|
||
|
||
</script>
|
||
|
||
<style type"text/css">
|
||
/* Turn on line numbering for every line of prettyprinted code, instead of every fifth line. */
|
||
li.L0, li.L1, li.L2, li.L3,
|
||
li.L5, li.L6, li.L7, li.L8 {
|
||
list-style-type: decimal !important;
|
||
}
|
||
</style>
|
||
|
||
<!-- Google Tag Manager -->
|
||
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||
})(window,document,'script','dataLayer','GTM-NT79GSP');</script>
|
||
<!-- End Google Tag Manager --></head><body class="body-9"><div data-collapse="medium" data-animation="default" data-duration="200" data-easing="ease" data-easing2="ease" role="banner" class="navbar-9 w-nav"><div class="section-15"><div class="container-12 w-container"><nav role="navigation" class="nav-menu-5 w-clearfix w-nav-menu"><a href="https://www.hostedgraphite.com/accounts/signup/" class="hg-nav-link right marketing get-started-ppc-button w-nav-link">Get started free</a><a href="https://www.hostedgraphite.com/accounts/login/" class="hg-nav-link right w-nav-link">Login</a><a href="https://www.hostedgraphite.com/enterprise" class="hg-nav-link w-nav-link">Enterprise</a><a href="https://www.hostedgraphite.com/pricing" class="hg-nav-link w-nav-link">Pricing</a><a href="https://www.hostedgraphite.com/docs/" class="hg-nav-link w-nav-link">Docs</a><a href="https://www.hostedgraphite.com/customers" class="hg-nav-link w-hidden-medium w-hidden-small w-hidden-tiny w-nav-link">Customers</a><a href="https://www.hostedgraphite.com/product" class="hg-nav-link w-nav-link">Features</a></nav><div class="menu-button-3 w-nav-button"><div class="w-icon-nav-menu"></div></div></div></div><div class="div-block-15"><a href="https://www.hostedgraphite.com" class="brand-4 w-nav-brand"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg" width="1088.5" alt="" class="image-52"/></a></div><div class="section-16"></div></div><div data-w-expand="category" style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png")" class="hero-blog"></div><div class="main-section"><div class="w-container"><div class="section-heading"><h1 class="blog-post-title">Our incident postmortem template</h1><div class="blog-date">August 15, 2019</div><a data-w-expand="category" href="/blog-categories/engineering" class="blog-category">Engineering</a><div class="full-divide"></div><div class="w-embed w-script"><script>
|
||
document.addEventListener("DOMContentLoaded", function() {
|
||
const tocContainer = document.createElement('div');
|
||
tocContainer.id = 'toc-container';
|
||
tocContainer.innerHTML = '<h2>Table of Contents</h2><ul id="toc"></ul>';
|
||
|
||
const sectionHeading = document.querySelector('.section-heading');
|
||
sectionHeading.insertAdjacentElement('afterend', tocContainer);
|
||
|
||
const toc = document.getElementById('toc');
|
||
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
|
||
|
||
headers.forEach((header) => {
|
||
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
|
||
header.id = id;
|
||
|
||
const li = document.createElement('li');
|
||
const a = document.createElement('a');
|
||
a.href = `#${id}`;
|
||
a.textContent = header.textContent;
|
||
li.appendChild(a);
|
||
|
||
toc.appendChild(li);
|
||
});
|
||
|
||
document.querySelectorAll('#toc a').forEach(anchor => {
|
||
anchor.addEventListener('click', function(e) {
|
||
e.preventDefault();
|
||
document.querySelector(this.getAttribute('href')).scrollIntoView({
|
||
behavior: 'smooth'
|
||
});
|
||
history.pushState(null, null, this.getAttribute('href'));
|
||
});
|
||
});
|
||
});
|
||
</script>
|
||
<style>
|
||
#toc-container {
|
||
text-align: start;
|
||
width: 85%;
|
||
margin: 0 auto 3em;
|
||
}
|
||
#toc-container h2 {
|
||
margin-top: 0;
|
||
}
|
||
#toc-container ul {
|
||
padding-left: 1em;
|
||
}
|
||
#toc-container li {
|
||
margin-bottom: 0.5em;
|
||
}
|
||
#toc li a {
|
||
text-decoration: none;
|
||
color: #2e2e2e;
|
||
}
|
||
#toc li a:hover {
|
||
text-decoration: underline;
|
||
}
|
||
</style></div></div><div class="w-condition-invisible w-embed"></div><div class="blog-content"><div class="blog-post w-richtext"><h3>In the final part of our SRE process series, we share our internal postmortem template with some pointers on the review process, what to include in each section, plus best practice examples. This follows <a href="https://www.hostedgraphite.com/blog/its-dead-jim-how-we-write-an-incident-postmortem">our recent postmortem guidelines post</a>, that looks at things to keep in mind before putting pen to paper. If you haven’t read it already, make sure to check that out first!</h3><h4>Writing as a two-step process</h4><p>At <a href="https://www.hostedgraphite.com">Hosted Graphite</a>, our customers are highly technical and appreciate a high level of detail. So for public-facing documents, we put considerable effort into getting the language and level of detail right. That said, the initial focus should be on compiling the bulk of its content in a complete and accurate way. Finessing the language comes later. </p><p>With this in mind, your first draft of the postmortem should focus on pulling together the information to include, and checking its accuracy. It should reflect what happened, explain why, and include all the lessons we have (collectively) learned from the incident. Once you've gotten positive feedback on the first draft, it’s time to work on the language and tone of the document to make it ready for public consumption (assuming that the goal is to publish this document externally). This lets us focus on getting the content right and separates concerns by focusing one review on content and accuracy, and a separate review on language and tone</p><h4>Reviewing the document</h4><p>Ideally, your first draft should be reviewed by all SREs involved in the incident, who may have their own contributions and suggestions for lessons learned. At least two SREs should sign off on your first draft before you move on to the second pass, which also needs to be approved by at least two SREs before it's considered ready to be published.</p><p>Once your postmortem has been reviewed (if this is a public postmortem) you can copy and paste the content of your postmortem document into <a href="https://status.hostedgraphite.com/">Statuspage</a>. Keep in mind that formatting adjustments might be necessary, so we should be extra careful that the formatting is right before publishing it.</p><p>Make sure to share it with the rest of SRE (and possibly the wider Hosted Graphite team). A postmortem document is only useful if the lessons it contains are shared with and understood by the rest of the team, so ask everyone to take some time to read the postmortem, and add some comments to it.</p><h4>The sections</h4><p><a href="https://docs.google.com/document/d/e/2PACX-1vSRnmu7Xj68WC4KKLNeCvcClhdKMOjShgGbUuL0-vBTd5GgdGoddfcCsQ5oc9Io6XG3hNQIzcvP5eNt/pub">This is a fairly simple template</a> as we'd much rather focus on the process and content than on structure. What follows is a quick explanation of each section with some tips and examples of what to include.</p><h2>Summary</h2><p>This is just a quick summary/headline of what the incident was: what happened and what the impact was. Usually no longer than one or two sentences. After reading this a person unfamiliar with this incident should know what the impact was and why they should keep reading.</p><h2>Background</h2><p>Keep in mind: postmortems are technical documents that are sometimes read by an audience that's not fully technical. Sometimes even a technical audience could have trouble keeping up with a postmortem document if they're lacking the necessary context. In this section, try to provide any context that may be necessary to fully understand the rest of the document. For example, a postmortem we wrote might reference our internal canaries, so a quick explanation of what they are is useful (and bonus points for linking to <a href="https://www.hostedgraphite.com/blog/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second">our blog post</a> about them).</p><p>Whenever possible, link to public documentation and/or a blog post. If we find ourselves explaining the same thing over and over it might be worth posting something to our blog that we can reference in the future.</p><h2>What happened?</h2><p>Here is where we tell the story of this incident. Everything that happened, including a timeline of events, goes here. This includes things that happened without our knowledge (i.e there was a traffic spike we weren't aware of) to alerts firing, to actions we took trying to mitigate the issue. For long-running incidents, the timeline sometimes gets a bit long. It makes sense to move it to its own section, that way people can refer to it if they want more detail while leaving the current section uncluttered.</p><p>Something that's valuable to add in this section is the specific information we had when we undertook a particular action. It's one thing saying that we added more capacity to a cluster but that didn't do anything, but it's way more useful to say that, for example, we saw increased load and latency on cluster X and therefore interpreted that as a capacity issue, deciding to increase capacity in the cluster. If later in the timeline that assumption is proven to be incorrect we'll know <strong>why</strong> we took that specific action and can work on making sure that we have more complete information next time.</p><h2>What went well?</h2><p>Not everything is terrible (I mean I guess...at least that's what people tell me!). Even during <a href="https://www.hostedgraphite.com/blog/spooky-action-at-a-distance-how-an-aws-outage-ate-our-load-balancer">the worst incident ever</a> some things will go well. Maybe <a href="https://www.hostedgraphite.com/">our monitoring</a> was very quick to react and alerted us just in time to prevent further impact. Maybe our service discovery routed around impacted nodes so the impact was only added latency instead of lost data.</p><p>For most incidents, there are circumstances that prevented it from being much worse. It's important to identify these circumstances, as they will increase our understanding of how the system works as much (if not more more) than figuring out the areas that made us particularly vulnerable.</p><h2>What went badly?</h2><p>There will be things that we're unhappy about: how the incident was handled, or where our tools/services let us down. This is a good place to point things we know that could have gone better. It’s also a good trigger to identify followup tasks for things to improve in the future.</p><p>There will also be factors that contributed to making this incident worse than we'd otherwise expect it to be. We want to be able to identify what these are, not just to fix, but so we can deepen our understanding of how our system works and the things that make less resilient.</p><h2>What are we going to do in the future?</h2><p>This is often the section our users care about most. What we've experienced is all well and good, but what have we actually <strong>learned</strong>, and more importantly, what are we going to do to make sure our users’ metrics don't go away again? The most important result of a postmortem is what we learn about our systems and how our people interact with them, but our users deserve to see those lessons become more specific actions.</p><p>With this in mind, any actions we plan to take in the future should have an open ticket to make sure they don't fall through the cracks. There should be a reference to the ticket in our internal document. For public postmortems, we don't include the full ticket but make reference to it (something like internal ticket: SRE-420). This will help us better track the progress of any follow-up items and enforce the fact that everything in this section needs to have a ticket attached to it. Those tickets should be clearly marked as incident followup (and also link to the original incident in our status page).</p><p><strong>Example:</strong></p><p>We will be sprinkling glitter over our servers to make them go faster (internal ticket: SRE-420)<br/></p><p>A clear sign of a bad postmortem is one where we don't know what to put in this section, or think that we don't have anything to include. A postmortem that results in no follow ups just means we haven't been looking hard enough. If we claim to have learned valuable lessons from an incident but can't come up with anything to do differently next time maybe we didn't learn the right lessons. Remember, "have better luck next time" or "don't make mistakes next time" is not a valid strategy.</p><h4>Things to keep in mind</h4><p>Overall the tone of our postmortems should be similar to the tone of our status page communications (after all, they are written by the same team with roughly the same audience in mind). That means anything that has been covered in our <a href="https://www.hostedgraphite.com/blog/how-to-write-a-status-page-update">Status page</a> <a href="https://www.hostedgraphite.com/blog/status-page-updates-its-all-about-timing">guidelines</a> regarding the kind of language that we use also applies to the postmortem.</p><p>When reviewing a postmortem try to focus more on content than language. It's always important to get the language right (in particular for public documents) but what matters is the process that resulted in that document and the lessons and actions derived from it. Remember, when you review a postmortem document you're reviewing the process that led to it as much as the final document itself.</p><h4>Other resources</h4><p>There are too many great resources out there to list, but the following should be considered required reading (or watching!) on the topic:</p><p><a href="https://landing.google.com/sre/book/chapters/postmortem-culture.html">Chapter 15 of the SRE book</a></p><p><a href="https://www.oreilly.com/ideas/the-infinite-hows">"The infinite hows" - John Allspaw</a></p><p><a href="https://www.youtube.com/watch?v=8DtzmV1jiyQ">Incidents as we Imagine Them Versus How They Actually Are - John Allspaw (video)</a></p><p><a href="http://web.mit.edu/2.75/resources/random/How%20Complex%20Systems%20Fail.pdf">How complex systems fail - Richard Cook</a></p><p><a href="https://www.adaptivecapacitylabs.com/blog/2018/10/08/the-multiple-audiences-and-purposes-of-post-incident-reviews/">The Multiple Audiences and Purposes of Post-Incident Reviews</a></p><p><a href="https://www.adaptivecapacitylabs.com/blog/2019/06/17/some-observations-on-the-messy-realities-of-incident-reviews/">Some Observations On the Messy Realities of Incident Reviews</a></p><p><a href="https://www.adaptivecapacitylabs.com/blog/2019/03/03/hindsight-and-sacrifice-decisions/">Hindsight and sacrifice decisions</a></p><p><br/></p><p><br/></p></div><div class="sticky-toc-container"><div class="cta-sidebar"><div class="text-block-31"><strong class="bold-text-15">Try Hosted Graphite now!</strong></div><p class="paragraph-12">Get Hosted Graphite free for 14 days. No credit card required.</p><a href="https://www.hostedgraphite.com/accounts/signup/?from=heroku_blogpost" class="cta-side-button w-button">Get Started</a></div><div class="toc w-embed w-script"><script>
|
||
document.addEventListener("DOMContentLoaded", function() {
|
||
function createSideToc(containerId) {
|
||
const container = document.querySelector(containerId);
|
||
|
||
// Create and append the title <h2>
|
||
const tocTitle = document.createElement('h2');
|
||
tocTitle.textContent = 'Table of Contents';
|
||
container.appendChild(tocTitle);
|
||
|
||
const toc = document.createElement('ul');
|
||
toc.className = 'toc';
|
||
|
||
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
|
||
const ids = new Set(); // To ensure unique IDs
|
||
|
||
headers.forEach((header) => {
|
||
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
|
||
header.id = id;
|
||
if (!ids.has(id)) {
|
||
ids.add(id);
|
||
|
||
const li = document.createElement('li');
|
||
const a = document.createElement('a');
|
||
a.href = `#${id}`;
|
||
a.textContent = header.textContent;
|
||
li.appendChild(a);
|
||
|
||
toc.appendChild(li);
|
||
}
|
||
});
|
||
|
||
container.appendChild(toc);
|
||
|
||
// Smooth scroll for the new TOC
|
||
toc.querySelectorAll('a').forEach(anchor => {
|
||
anchor.addEventListener('click', function(e) {
|
||
e.preventDefault();
|
||
document.querySelector(this.getAttribute('href')).scrollIntoView({
|
||
behavior: 'smooth'
|
||
});
|
||
history.pushState(null, null, this.getAttribute('href'));
|
||
});
|
||
});
|
||
}
|
||
|
||
// Create the sticky TOC in the sidebar
|
||
createSideToc('.sticky-toc-container');
|
||
});
|
||
</script>
|
||
|
||
<style>
|
||
/* Specific Styling for Sticky TOC */
|
||
.sticky-toc-container {
|
||
position: sticky;
|
||
top: 110px;
|
||
width: 340px;
|
||
height: 100%;
|
||
overflow-y: auto;
|
||
padding: 15px;
|
||
border-radius: 8px;
|
||
font-family: Arial, sans-serif;
|
||
font-size: 14px;
|
||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
|
||
text-align: left;
|
||
}
|
||
|
||
.sticky-toc-container h2 {
|
||
margin-top: 0;
|
||
font-size: 18px;
|
||
color: #2e2e2e;
|
||
}
|
||
|
||
.sticky-toc-container ul {
|
||
list-style-type: none;
|
||
padding-left: 0;
|
||
margin: 0;
|
||
}
|
||
|
||
.sticky-toc-container li {
|
||
margin-bottom: 0.6em;
|
||
}
|
||
|
||
.sticky-toc-container a {
|
||
text-decoration: none;
|
||
color: #8e8b8b;
|
||
}
|
||
|
||
.sticky-toc-container a:hover {
|
||
text-decoration: underline;
|
||
}
|
||
|
||
/* Make sure the TOC has a scroll bar when needed */
|
||
.toc {
|
||
max-height: 35vh;
|
||
overflow-y: auto;
|
||
}
|
||
|
||
/* Media Query to Hide sticky-toc-container on smaller screens */
|
||
@media (max-width: 1215px) {
|
||
.sticky-toc-container {
|
||
display: none;
|
||
}
|
||
}
|
||
</style></div></div></div><div class="w-condition-invisible w-embed w-script"><script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js"></script>
|
||
<pre class="prettyprint">
|
||
|
||
</pre></div><div class="div-block-2"><div class="full-divide"></div><div class="author-wrapper"><a href="/blog-authors/fran-garcia" data-w-expand="authors" class="author-name">Fran Garcia</a><div class="smallest-divider"></div><div data-w-expand="authors" class="author-bio w-richtext"><p>SRE at Hosted Graphite.</p></div><div><a href="https://twitter.com/hostedgraphite" data-w-expand="authors" class="social-link w-inline-block"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a57aa7ad1fa2300015ca29a_social-18.svg" width="68" alt="" class="image-2"/></a><a data-w-expand="authors" href="#" class="social-link w-inline-block"></a><a href="#" data-w-expand="authors" class="social-link w-inline-block"></a></div></div></div></div></div><div class="main-section gray"><div class="w-container"><div class="section-heading"><h2 class="heading-6">Related Posts</h2><div class="med-divider"></div></div><div class="w-dyn-list"><div role="list" class="w-clearfix w-dyn-items w-row"><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/hosted-graphite-isnt-graphite" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5dd37e9fe192cd2ea5b8af68_Screen%20Shot%202019-11-19%20at%202.14.02%20PM.jpg")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">HostedGraphite</div><div class="preview-text">Hosted Graphite improves upon standard Graphite. Take a look at how we do this and how Hosted Graphite gives your company better functionality. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Shevaun Frazier</div><div class="thumbnail-date">Nov 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/new-developer-onboarding" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5bfea01e7ffa1b30f88f19c0_Blog-background-organge.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">New Developer Onboarding</div><div class="preview-text">Discussions about onboarding tend to revolve around new hires, for obvious reasons, but the process is important for everyone: while the new developer learns the most important aspects of team and company culture, the team has an opportunity to learn new ideas from a fresh pair of eyes. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Heather Wiencko</div><div class="thumbnail-date">Sep 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/pug-life-how-we-run-a-grafana-instance-for-each-user-with-docker" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5d643d90b1c59597a144484a_pug-header.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">PUG Life: How we run a Grafana instance for each user with Docker </div><div class="preview-text">The story of Hosted Grafana at Hosted Graphite, and how we run multiple instances of Grafana. Start monitoring with Hosted Graphite and use our Grafana dashboards directly in-platform. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciarán Finn</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/its-dead-jim-how-we-write-an-incident-postmortem" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">"It's dead, Jim": How we write an incident postmortem</div><div class="preview-text">How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Jul 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/surviving-on-call-tips-from-a-hosted-graphite-sre" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c48a4405b91aaf448087677_daveblog-header.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Surviving On-Call: Tips from a Hosted Graphite SRE</div><div class="preview-text">Get an insight into what Hosted Graphite's SRE Dave Fennell has learned when being On-Call. Read the tips that he has to offer to make the On-Call experience a positive one.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Dave Fennell</div><div class="thumbnail-date">Jan 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/a-victim-of-its-own-popularity-scaling-our-cloudwatch-integration" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">A victim of its own popularity: Scaling our CloudWatch integration</div><div class="preview-text">Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Egan</div><div class="thumbnail-date">Dec 2018</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/deadlines-lies-and-videotape-the-tale-of-a-grpc-bug" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Deadlines lies and videotape: The tale of a gRPC bug</div><div class="preview-text">If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released).</div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Gaffney</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/status-page-updates-its-all-about-timing" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Status page updates: It’s all about timing</div><div class="preview-text">Check out part 2 of Hosted Graphite's SRE process on how to handing status page updates. Learn more in detail on the responsibilities of communication, duration of updating the page, and who’s responsible for certain tasks.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Oct 2018</div></div></a></div></div></div></div></div><div class="main-section dark"><div class="container-13 w-container"><div class="section-heading"><h2 class="white">See why thousands of engineers trust Hosted Graphite with their monitoring</h2><div class="med-divider"></div></div><a href="https://www.hostedgraphite.com/accounts/signup/" class="get-started-ppc-button w-button">START A FREE TRIAL</a><div class="div-block-14 w-hidden-medium w-hidden-small w-hidden-tiny"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png" width="120" sizes="(max-width: 991px) 100vw, 120px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1600.png 1600w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png 2000w" alt="" class="image-5"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png" width="190" sizes="(max-width: 991px) 100vw, 190px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png 1280w" alt="" class="image-3"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1)-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png 620w" alt="" class="image-4"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png 800w" alt="" class="image-6"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd52f6b273e0001a3d86c_Tableau%20(1).svg" alt="" class="image-7"/></div></div><div class="footer-section"><div class="w-container"><div>© 2024 Metricfire Limited. <br/>All Rights Reserved.<br/><br/><br/>5940 S Rainbow Blvd Ste 400<br/>Las Vegas, NV 89118-2507<br/>United States</div></div></div></div><div class="w-embed"></div><div class="w-embed w-script"><script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "Article",
|
||
"mainEntityOfPage": {
|
||
"@type": "WebPage",
|
||
"@id": "https://www.hostedgraphite.com/blog/incident-postmortem-template"
|
||
},
|
||
"headline": "Our incident postmortem template",
|
||
"image": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png",
|
||
"thumbnailUrl": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png",
|
||
"author": {
|
||
"@type": "Person",
|
||
"name": "Fran Garcia",
|
||
"url": "fran-garcia"
|
||
},
|
||
"publisher": {
|
||
"@type": "Organization",
|
||
"name": "Metricfire Limited",
|
||
"logo": {
|
||
"@type": "ImageObject",
|
||
"url": "https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg"
|
||
}
|
||
},
|
||
"datePublished": "Aug 15, 2019",
|
||
"dateModified": "Sep 15, 2021",
|
||
"description": "Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples."
|
||
}
|
||
</script>
|
||
|
||
<script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "SoftwareApplication",
|
||
"applicationCategory": "BusinessApplication",
|
||
"name": "Hosted Graphite",
|
||
"description": "Hosted Graphite offers powerful monitoring and alerting for various business applications, providing integrations with multiple data sources, a Grafana UI, and great customer support.",
|
||
"review": [
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Cecily G." },
|
||
"reviewBody": "This system is the tool every engineer needs to be successful. Easy to use and captures metrics that no other system has successfully been able to pull for us.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Christopher C." },
|
||
"reviewBody": "Great product suite, outstanding customer service! Running a distributed data pipeline is challenging to keep healthy, and Hosted Graphite has been an instrumental partner for monitoring system health and getting ahead of issues.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "UC" },
|
||
"reviewBody": "A great service with excellent support. Simple to use, inexpensive, and great support.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Brendan C." },
|
||
"reviewBody": "High-quality metrics hosting. Hosted Graphite enables us to instrument our products and processes efficiently. We can proactively monitor for issues and use dashboards to identify more minor performance problems before they become major ones.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Jacobson M." },
|
||
"reviewBody": "Perfect for remote device analytics. Hosted graphite makes it easy for me to keep track of my remote devices being used by my customers. I can't always be on site, so this allows easy tracking of device details.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "James C." },
|
||
"reviewBody": "The easiest way for DevOps to incorporate Kubernetes without having to suffer a paradigm shift away from using Graphite. Prometheus, while academically good, forces security worst practices which were unacceptable for us as a SOC2 compliant organization. Graphite is battle-proven and already integrated, so Hosted Graphite allowed us to do what we needed without drastic changes.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
|
||
"datePublished": "2021-03-19"
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Anne M." },
|
||
"reviewBody": "Offers great hosting service, Hosted Graphite is quite easy to use and highly secure. I like how they offer the trial version to evaluate if the solution fits your needs. In terms of setup, it's easy with minimal registration details and flexible pricing.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
|
||
"datePublished": "2019-02-20"
|
||
}
|
||
],
|
||
"operatingSystem": "All major operating systems",
|
||
"aggregateRating": {
|
||
"@type": "AggregateRating",
|
||
"reviewCount": 7,
|
||
"ratingValue": 4.85,
|
||
"bestRating": 5,
|
||
"worstRating": 1
|
||
},
|
||
"offers": {
|
||
"@type": "Offer",
|
||
"price": 19.00,
|
||
"priceCurrency": "USD",
|
||
"availability": "https://schema.org/InStock",
|
||
"url": "https://hostedgraphite.com",
|
||
"seller": {
|
||
"@type": "Organization",
|
||
"name": "Hosted Graphite"
|
||
}
|
||
}
|
||
}
|
||
</script></div><script src="https://d3e54v103j8qbb.cloudfront.net/js/jquery-3.5.1.min.dc5e7f18c8.js?site=5a57aa76d1fa2300015ca244" type="text/javascript" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.36b8fb49256177c8.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.83de43c5e9eda212.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.54e1a7f0.3d4248ede2db61fe.js" type="text/javascript"></script><!-- Google Tag Manager (noscript) -->
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-NT79GSP"
|
||
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
<!-- End Google Tag Manager (noscript) --><script>
|
||
img = document.querySelectorAll('.w-embed');
|
||
img.forEach(element => element.style.textAlign = 'center');
|
||
</script></body></html> |