Files
nexus/sreweekly/articles/89/02-adventures-in-fault-tolerant-alerting-with-python.html
2026-09-12 17:23:01 +08:00

367 lines
44 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><!-- Last Published: Wed Jul 02 2025 01:43:30 GMT+0000 (Coordinated Universal Time) --><html data-wf-domain="webflow.hostedgraphite.com" data-wf-page="5ce3e821b5d6678199f31df7" data-wf-site="5a57aa76d1fa2300015ca244" data-wf-collection="5ce3e821b5d6674c4bf31e14" data-wf-item-slug="adventures-in-fault-tolerant-alerting-with-python"><head><meta charset="utf-8"/><link href="https://cdn.prod.website-files.com" rel="preconnect" crossorigin="anonymous"/><title>Adventures in fault tolerant alerting with Python</title><meta content="Learn about Hosted Graphite&#x27;s presentation at Python Ireland&#x27;s April meetup using PySyncObj. See what type of tech our engineers have chosen and why." name="description"/><meta content="Adventures in fault tolerant alerting with Python" property="og:title"/><meta content="Learn about Hosted Graphite&#x27;s presentation at Python Ireland&#x27;s April meetup using PySyncObj. See what type of tech our engineers have chosen and why." property="og:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad0843d91f6b21c62e16e15_jcb_flowers.jpg" property="og:image"/><meta content="Adventures in fault tolerant alerting with Python" name="twitter:title"/><meta content="Learn about Hosted Graphite&#x27;s presentation at Python Ireland&#x27;s April meetup using PySyncObj. See what type of tech our engineers have chosen and why." name="twitter:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad0843d91f6b21c62e16e15_jcb_flowers.jpg" name="twitter:image"/><meta property="og:type" content="website"/><meta content="summary_large_image" name="twitter:card"/><meta content="width=device-width, initial-scale=1" name="viewport"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/css/hosted-graphite-blog.webflow.shared.f88b646c4.min.css" rel="stylesheet" type="text/css"/><link href="https://fonts.googleapis.com" rel="preconnect"/><link href="https://fonts.gstatic.com" rel="preconnect" crossorigin="anonymous"/><script src="https://ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js" type="text/javascript"></script><script type="text/javascript">WebFont.load({ google: { families: ["Open Sans:300,300italic,400,400italic,600,600italic,700,700italic,800,800italic","Inconsolata:400,700","Montserrat:100,100italic,200,200italic,300,300italic,400,400italic,500,500italic,600,600italic,700,700italic,800,800italic,900,900italic","Lato:100,100italic,300,300italic,400,400italic,700,700italic,900,900italic","Droid Sans:400,700","Vollkorn:400,400italic,700,700italic","Ubuntu:300,300italic,400,400italic,500,500italic,700,700italic","Lora:regular,italic,700","Oxygen:300,regular,700","Source Sans Pro:200,200italic,300,300italic,regular,600,600italic,700,700italic,900,900italic"] }});</script><script type="text/javascript">!function(o,c){var n=c.documentElement,t=" w-mod-";n.className+=t+"js",("ontouchstart"in o||o.DocumentTouch&&c instanceof DocumentTouch)&&(n.className+=t+"touch")}(window,document);</script><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f272bbb5a83b7b3239ad6_32.png" rel="shortcut icon" type="image/x-icon"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f27323f9c40d523c9bfcc_256.png" rel="apple-touch-icon"/><script src="https://cdn.jsdelivr.net/npm/code-prettify@0.1.0/src/prettify.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script type="text/javascript">
var embeddify = function(){
$("h6").each(function(index){
var h6 = $(this);
var url = h6.text();
console.log("Embeddifying " + url);
h6.hide();
var embed_message = $('<pre>Embedding <a href="'+url+'" target="_blank">some code</a>, one moment...</pre>');
embed_message.insertAfter(h6);
$.ajax(url, {
success: function(content){
$('<?prettify linenums=true?><pre class="prettyprint">'+content+'</pre>').insertAfter(h6);
embed_message.hide();
PR.prettyPrint();
},
error: function(jqXHR, textStatus, errorThrown){
embed_message.html('Tried to fetch <a href="'+url+'" target="_blank">some code from GitHub</a>, but it failed with an error: "'+errorThrown+'"');
}
});
});
};
var webflow_removed = false;
var attempts_remaining = 10;
function dewebflow() {
/* Temporarily remove this badge while we're sorting out
serving this from the right domains so the hosting provider
will remove the badge properly. */
var badges = $("a.w-webflow-badge");
if(badges.length == 1)
{
$("a.w-webflow-badge").remove();
webflow_removed = true;
} else {
attempts_remaining--;
}
if(!webflow_removed || attempts_remaining > 0)
setTimeout(dewebflow, 500);
}
$(document).ready(embeddify);
$(document).ready(dewebflow);
</script>
<style type"text/css">
/* Turn on line numbering for every line of prettyprinted code, instead of every fifth line. */
li.L0, li.L1, li.L2, li.L3,
li.L5, li.L6, li.L7, li.L8 {
list-style-type: decimal !important;
}
</style>
<!-- Google Tag Manager -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer','GTM-NT79GSP');</script>
<!-- End Google Tag Manager --></head><body class="body-9"><div data-collapse="medium" data-animation="default" data-duration="200" data-easing="ease" data-easing2="ease" role="banner" class="navbar-9 w-nav"><div class="section-15"><div class="container-12 w-container"><nav role="navigation" class="nav-menu-5 w-clearfix w-nav-menu"><a href="https://www.hostedgraphite.com/accounts/signup/" class="hg-nav-link right marketing get-started-ppc-button w-nav-link">Get started free</a><a href="https://www.hostedgraphite.com/accounts/login/" class="hg-nav-link right w-nav-link">Login</a><a href="https://www.hostedgraphite.com/enterprise" class="hg-nav-link w-nav-link">Enterprise</a><a href="https://www.hostedgraphite.com/pricing" class="hg-nav-link w-nav-link">Pricing</a><a href="https://www.hostedgraphite.com/docs/" class="hg-nav-link w-nav-link">Docs</a><a href="https://www.hostedgraphite.com/customers" class="hg-nav-link w-hidden-medium w-hidden-small w-hidden-tiny w-nav-link">Customers</a><a href="https://www.hostedgraphite.com/product" class="hg-nav-link w-nav-link">Features</a></nav><div class="menu-button-3 w-nav-button"><div class="w-icon-nav-menu"></div></div></div></div><div class="div-block-15"><a href="https://www.hostedgraphite.com" class="brand-4 w-nav-brand"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg" width="1088.5" alt="" class="image-52"/></a></div><div class="section-16"></div></div><div data-w-expand="category" style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad0843d91f6b21c62e16e15_jcb_flowers.jpg&quot;)" class="hero-blog"></div><div class="main-section"><div class="w-container"><div class="section-heading"><h1 class="blog-post-title">Adventures in fault tolerant alerting with Python</h1><div class="blog-date">May 5, 2017</div><a data-w-expand="category" href="/blog-categories/engineering" class="blog-category">Engineering</a><div class="full-divide"></div><div class="w-embed w-script"><script>
document.addEventListener("DOMContentLoaded", function() {
const tocContainer = document.createElement('div');
tocContainer.id = 'toc-container';
tocContainer.innerHTML = '<h2>Table of Contents</h2><ul id="toc"></ul>';
const sectionHeading = document.querySelector('.section-heading');
sectionHeading.insertAdjacentElement('afterend', tocContainer);
const toc = document.getElementById('toc');
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
headers.forEach((header) => {
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
header.id = id;
const li = document.createElement('li');
const a = document.createElement('a');
a.href = `#${id}`;
a.textContent = header.textContent;
li.appendChild(a);
toc.appendChild(li);
});
document.querySelectorAll('#toc a').forEach(anchor => {
anchor.addEventListener('click', function(e) {
e.preventDefault();
document.querySelector(this.getAttribute('href')).scrollIntoView({
behavior: 'smooth'
});
history.pushState(null, null, this.getAttribute('href'));
});
});
});
</script>
<style>
#toc-container {
text-align: start;
width: 85%;
margin: 0 auto 3em;
}
#toc-container h2 {
margin-top: 0;
}
#toc-container ul {
padding-left: 1em;
}
#toc-container li {
margin-bottom: 0.5em;
}
#toc li a {
text-decoration: none;
color: #2e2e2e;
}
#toc li a:hover {
text-decoration: underline;
}
</style></div></div><div class="w-condition-invisible w-embed"></div><div class="blog-content"><div class="blog-post w-richtext"><p>We gave a presentation a couple of weeks ago <a href="https://www.meetup.com/pythonireland/events/238463241/" target="_blank">Python Ireland&#x27;s April meetup</a> where we described our experiences with <a href="https://github.com/bakwc/PySyncObj" target="_blank">PySyncObj</a>, a relatively new but solid library for building fault tolerant distributed systems in Python. Most of the services that run Hosted Graphite are built in Python, and this includes our alerting system. While that talk wasn&#x27;t recorded, this blog post discusses what we did, the tech we chose, and why.</p><h3>A monitoring system without alerting</h3><p>We&#x27;ve been running Hosted Graphite as a big distributed custom time-series database since 2012, and once we mastered the monitoring side of things, the next obvious step was an integrated alerting system.</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/alerts1.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/alerts1.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/alerts1.webp" alt="A screenshot of the Beta version of Hosted Graphite's Alerts tab">
</picture></div><p>‍</p><p>We released a beta version of the alerting feature in 2016, and we&#x27;ve kept it technically in &quot;beta&quot; for the same reasons Google kept GMail in beta for more than five years: people are using it for real work, we&#x27;re supporting it, but we know we still have work to do and so there&#x27;s still a &quot;beta&quot; label on it for now.</p><p>One of the biggest things that was holding us back from calling the alerting feature fully baked was the failover characteristics. Almost everything else at Hosted Graphite is a distributed, fault-tolerant distributed system, and we knew we needed the same for an alerting system. After all, we&#x27;re on-call 24/7 for it!</p><h3>Keeping state</h3><p>Some parts of the alerting system are easy to do failover for because they keep no state, like the streaming and polling monitors in this slide:</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://s3.console.aws.amazon.com/s3/object/hgblogimg?region=us-east-2&prefix=adventures-in-fault-tolerant-alerting-with-python/beta2.webp" type="image/webp">
<source srcset="https://s3.console.aws.amazon.com/s3/object/hgblogimg?region=us-east-2&prefix=adventures-in-fault-tolerant-alerting-with-python/beta2.jpg" type="image/jpeg">
<img src="https://s3.console.aws.amazon.com/s3/object/hgblogimg?region=us-east-2&prefix=adventures-in-fault-tolerant-alerting-with-python/beta2.webp" alt="A diagram showing the inner workings of how metrics from the customer are being monitored and pointing to the alerter in beta">
</picture></div><p>‍</p><p>‍</p><p>The trouble starts when we consider the state kept by the Alerter component. The Alerter receives a series of &quot;OK&quot;, &quot;OK&quot;, &quot;not OK!&quot; messages from the streaming and polling monitors, and needs to keep a lot of those in memory in order to make decisions about which notifications to send, if we need to wait a few more minutes to satisfy a &quot;must be alerting for more than five minutes&quot; constraint, when we&#x27;ve already sent enough notifications, and so on. Discarding the alerter state each time there is a failure would cause a pretty poor user experience with duplicate, delayed and missing notifications and we can&#x27;t tolerate that.</p><h3>Fine - just duplicate all the traffic!</h3><p>We considered just duplicating all the traffic to keep multiple alerters in sync with checkpointing to allow new nodes to catch up. <a href="https://kafka.apache.org/" target="_blank">Kafka</a> looked like a pretty good option at first, but it didn&#x27;t fare well in a review with the SRE team.</p><p>Here&#x27;s why:</p><ol role="list"><li>The data volume flowing to the alerter nodes isn&#x27;t huge - only a few thousand messages per second.</li><li>We run everything on bare metal hardware for high performance and avoiding the <a href="https://en.wikipedia.org/wiki/Cloud_computing_issues#Performance_interference_and_noisy_neighbors" target="_blank">noisy neighbour problem</a>.</li><li>While most of our users are served by our shared production infrastructure, we have many users whose needs demand a dedicated cluster. This isn&#x27;t a problem in itself, but it does mean that we&#x27;ll need to duplicate the services for each cluster, no matter how much load that customer puts on us.</li><li>Kafka requires <a href="https://zookeeper.apache.org/" target="_blank">Zookeeper</a> for cluster management too, so that&#x27;s yet another thing needing several machines. We already run <a href="https://github.com/coreos/etcd" target="_blank">etcd</a> anyway, and we don&#x27;t want to run two services that do similar things.</li></ol><p>Basically, Kafka is designed for bigger data volumes than providing high availability for this one service, and it didn&#x27;t make a lot of sense to run it here.</p><h3>Re-evaluating: what do we actually want?</h3><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/bulldozer.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/bulldozer.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/bulldozer.webp" alt="An image of an elder man holding a yellow flower, standing on the side of a small bulldozer covered in flowers">
</picture></div><p>‍</p><p>We zoomed out a bit and reconsidered. What failover properties do we want the alerting system to have? Here&#x27;s what we settled on:</p><ul role="list"><li>Nodes should exchange state among themselves.</li><li>Nodes should detect failure of a peer.</li><li>Nodes should figure out a new primary after the old one fails.</li><li>Nodes should assemble themselves into a new cluster automatically when we provision them.</li><li>... all without waking anyone up.</li></ul><h3>PySyncObj - pure Python Raft consensus</h3><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/pysyncobj.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/pysyncobj.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/pysyncobj.webp" alt="A screenshot of PySyncObj, a Python library, information page">
</picture></div><p>‍</p><p><a href="https://github.com/bakwc/PySyncObj" target="_blank">PySyncObj</a> is a pure Python library that solves all the requirements except the basic service discovery one, which we figured we could handle with etcd. PySyncObj allows us to define an object that is replicated among all nodes in the cluster, and it takes care of all the details It uses the <a href="https://raft.github.io/" target="_blank">Raft protocol</a> to replicate the data, elect a leader, detect when nodes have failed, deal with network partitions, etc.</p><p>Following Python&#x27;s &quot;batteries included&quot; philosophy, PySyncObj includes a set of basic replicated data types: a counter, dictionary, list, and a set. You can also define custom replicated objects with a couple of extra lines and a decorator, which is pretty awesome.</p><p>Here&#x27;s a simple example of how easy it is to replicate a dictionary across three servers with PySycObj:</p><h6>https://gist.githubusercontent.com/HG-blog/92cd27b159b63fb750a76b178d82898f/raw/d44b561c75aa643ac588d74c94ffc0ccda630e05/gistfile1.txt<br/></h6><p>All the distributed Raft magic is done for you. Pretty fantastic. For more examples, check out other examples on the <a href="https://github.com/bakwc/PySyncObj#batteries" target="_blank">PySyncObj github page</a>.</p><h3>Monitoring the new Raft cluster</h3><p>PySyncObj exports a bunch of data about the state of the Raft cluster with the <em>getStatus()</em> function:</p><h6>https://gist.githubusercontent.com/HG-blog/f7899cf640b4a64d48f7da4004901bf2/raw/31ca4adaf8349ebb3d5e9361383182b2f17987cb/gistfile1.txt<br/></h6><p>To monitor this data while we built confidence in the new service before rolling it out, we just fired it into Hosted Graphite using the dead simple <a href="https://github.com/derpston/python-graphiteudp" target="_blank">graphiteudp library</a>:</p><h6>https://gist.githubusercontent.com/HG-blog/015dcc21ffc2bd6c579f9166445dd5f6/raw/2ede63d083f9825203c911daba4d89257bfd427f/gistfile1.txt<br/></h6><p>The result is a pretty dashboard detailing the Raft cluster state for production and staging environments:</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/hg6.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/hg6.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/hg6.webp" alt="A screenshot of a Hosted Graphite dahsboard monitoring the new Raft cluster using PySyncObj">
</picture></div><p>‍</p><h3>Chaos monkey testing</h3><p>At this stage, we were pretty confident that this was the right route to take, but we wanted more. The only way to make your failover handling works is to continuously make it fail and see how it reacts, and that&#x27;s exactly what we did. We stood up a full-size test cluster and ran a copy of all of the alerting traffic from the production environment through it. The output was discarded so customers didn&#x27;t get duplicate alerts, but we were collecting all the same metrics about the output that we usually do.</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/hg+7.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/hg+7.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/hg+7.webp" alt="A screenshot of a Hosted Graphite dashboard panel monitoring the Leader Status of the Raft Cluster">
</picture></div><p>‍</p><p>In the style of the Netflix chaos monkey, we wrote a small tool to wait a random amount of time, and then restart one of the nodes without warning. We ran this for several days and it worked flawlessly - every time we lost a node, another was elected the new leader in a few seconds, the restarted node recovered the state it lost, and the overall output of the cluster (alerts checked, notifications that would have been sent, etc) was steady throughout.</p><h3>Emergency options</h3><p>Distributed systems are pretty fantastic when they work well, but it&#x27;s easy to get into a state where you just want a simple option to take control of a situation. In case we ever need it, we built in a &#x27;standalone&#x27; mode, where all the state is persisted to disk regularly. During a particularly chaotic incident, we have the option of quickly bypassing the clustered automatic failover functionality while we get a situation under control. We hope never to need it, but it&#x27;s nice that it&#x27;s there.</p><h3>Current state</h3><p>After a couple of weeks building confidence in the new alerter cluster, we quietly launched it to... no fanfare at all and no user interruption during the migration. In the weeks since launch, it has already faced several production incidents and fared well in all but one: <a href="https://status.hostedgraphite.com/incidents/8stmfdw1c9hh" target="_blank">a small bug</a> that took the alerting service out of the 99.9% SLA for the month. Oops. Despite all the testing, there&#x27;ll always be something to catch you out.</p><p>After more than a year of effort by our dev and SRE teams, the alerting service will soon have the &#x27;beta&#x27; label torn off and this automatic failover work already in production forms a crucial piece of that.</p><p>Here&#x27;s what it looks like now:</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/beta9.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/beta9.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/adventures-in-fault-tolerant-alerting-with-python/beta9.webp" alt="An updated version of Hosted Graphite's digram displaying a similar concept to the original beta diagram except having multiple alerters in a cluster">
</picture></div><p>‍</p><p>Service discovery for the alerter cluster is handled by etcd, and PySyncObj takes care of all other cluster operations. We&#x27;re pretty happy with this - we&#x27;re able to avoid waking someone on the SRE team for losing a machine or two, resize and upgrade the cluster without maintenance periods, and scale it up and down to trade off SLA requirements against cost, all of which is pretty powerful stuff.</p><p> </p><p>To learn more, visit <a href="https://www.hostedgraphite.com" target="_blank">hostedgraphite.com</a> or <a href="https://www.twitter.com/HostedGraphite" target="_blank">follow us on twitter</a>.</p></div><div class="sticky-toc-container"><div class="cta-sidebar"><div class="text-block-31"><strong class="bold-text-15">Try Hosted Graphite now!</strong></div><p class="paragraph-12">Get Hosted Graphite free for 14 days. No credit card required.</p><a href="https://www.hostedgraphite.com/accounts/signup/?from=heroku_blogpost" class="cta-side-button w-button">Get Started</a></div><div class="toc w-embed w-script"><script>
document.addEventListener("DOMContentLoaded", function() {
function createSideToc(containerId) {
const container = document.querySelector(containerId);
// Create and append the title <h2>
const tocTitle = document.createElement('h2');
tocTitle.textContent = 'Table of Contents';
container.appendChild(tocTitle);
const toc = document.createElement('ul');
toc.className = 'toc';
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
const ids = new Set(); // To ensure unique IDs
headers.forEach((header) => {
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
header.id = id;
if (!ids.has(id)) {
ids.add(id);
const li = document.createElement('li');
const a = document.createElement('a');
a.href = `#${id}`;
a.textContent = header.textContent;
li.appendChild(a);
toc.appendChild(li);
}
});
container.appendChild(toc);
// Smooth scroll for the new TOC
toc.querySelectorAll('a').forEach(anchor => {
anchor.addEventListener('click', function(e) {
e.preventDefault();
document.querySelector(this.getAttribute('href')).scrollIntoView({
behavior: 'smooth'
});
history.pushState(null, null, this.getAttribute('href'));
});
});
}
// Create the sticky TOC in the sidebar
createSideToc('.sticky-toc-container');
});
</script>
<style>
/* Specific Styling for Sticky TOC */
.sticky-toc-container {
position: sticky;
top: 110px;
width: 340px;
height: 100%;
overflow-y: auto;
padding: 15px;
border-radius: 8px;
font-family: Arial, sans-serif;
font-size: 14px;
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
text-align: left;
}
.sticky-toc-container h2 {
margin-top: 0;
font-size: 18px;
color: #2e2e2e;
}
.sticky-toc-container ul {
list-style-type: none;
padding-left: 0;
margin: 0;
}
.sticky-toc-container li {
margin-bottom: 0.6em;
}
.sticky-toc-container a {
text-decoration: none;
color: #8e8b8b;
}
.sticky-toc-container a:hover {
text-decoration: underline;
}
/* Make sure the TOC has a scroll bar when needed */
.toc {
max-height: 35vh;
overflow-y: auto;
}
/* Media Query to Hide sticky-toc-container on smaller screens */
@media (max-width: 1215px) {
.sticky-toc-container {
display: none;
}
}
</style></div></div></div><div class="w-condition-invisible w-embed w-script"><script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js"></script>
<pre class="prettyprint">
</pre></div><div class="div-block-2"><div class="full-divide"></div><div class="author-wrapper"><a href="/blog-authors/charlie-von-metzradt" data-w-expand="authors" class="author-name">Charlie von Metzradt</a><div class="smallest-divider"></div><div data-w-expand="authors" class="author-bio w-richtext"><p>Co-founder of Hosted Graphite, Software Engineer, huge nerd.</p></div><div><a href="https://twitter.com/hostedgraphite" data-w-expand="authors" class="social-link w-inline-block"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a57aa7ad1fa2300015ca29a_social-18.svg" width="68" alt="" class="image-2"/></a><a data-w-expand="authors" href="#" class="social-link w-inline-block"></a><a href="#" data-w-expand="authors" class="social-link w-inline-block"></a></div></div></div></div></div><div class="main-section gray"><div class="w-container"><div class="section-heading"><h2 class="heading-6">Related Posts</h2><div class="med-divider"></div></div><div class="w-dyn-list"><div role="list" class="w-clearfix w-dyn-items w-row"><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/hosted-graphite-isnt-graphite" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5dd37e9fe192cd2ea5b8af68_Screen%20Shot%202019-11-19%20at%202.14.02%20PM.jpg&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">HostedGraphite</div><div class="preview-text">Hosted Graphite improves upon standard Graphite. Take a look at how we do this and how Hosted Graphite gives your company better functionality. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Shevaun Frazier</div><div class="thumbnail-date">Nov 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/new-developer-onboarding" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5bfea01e7ffa1b30f88f19c0_Blog-background-organge.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">New Developer Onboarding</div><div class="preview-text">Discussions about onboarding tend to revolve around new hires, for obvious reasons, but the process is important for everyone: while the new developer learns the most important aspects of team and company culture, the team has an opportunity to learn new ideas from a fresh pair of eyes. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Heather Wiencko</div><div class="thumbnail-date">Sep 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/pug-life-how-we-run-a-grafana-instance-for-each-user-with-docker" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5d643d90b1c59597a144484a_pug-header.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">PUG Life: How we run a Grafana instance for each user with Docker </div><div class="preview-text">The story of Hosted Grafana at Hosted Graphite, and how we run multiple instances of Grafana. Start monitoring with Hosted Graphite and use our Grafana dashboards directly in-platform. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciarán Finn</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/incident-postmortem-template" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Our incident postmortem template</div><div class="preview-text">Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Aug 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/its-dead-jim-how-we-write-an-incident-postmortem" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">&quot;It&#x27;s dead, Jim&quot;: How we write an incident postmortem</div><div class="preview-text">How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Jul 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/surviving-on-call-tips-from-a-hosted-graphite-sre" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c48a4405b91aaf448087677_daveblog-header.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Surviving On-Call: Tips from a Hosted Graphite SRE</div><div class="preview-text">Get an insight into what Hosted Graphite&#x27;s SRE Dave Fennell has learned when being On-Call. Read the tips that he has to offer to make the On-Call experience a positive one.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Dave Fennell</div><div class="thumbnail-date">Jan 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/a-victim-of-its-own-popularity-scaling-our-cloudwatch-integration" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">A victim of its own popularity: Scaling our CloudWatch integration</div><div class="preview-text">Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Egan</div><div class="thumbnail-date">Dec 2018</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/deadlines-lies-and-videotape-the-tale-of-a-grpc-bug" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Deadlines lies and videotape: The tale of a gRPC bug</div><div class="preview-text">If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released).</div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Gaffney</div><div class="thumbnail-date">Nov 2020</div></div></a></div></div></div></div></div><div class="main-section dark"><div class="container-13 w-container"><div class="section-heading"><h2 class="white">See why thousands of engineers trust Hosted Graphite with their monitoring</h2><div class="med-divider"></div></div><a href="https://www.hostedgraphite.com/accounts/signup/" class="get-started-ppc-button w-button">START A FREE TRIAL</a><div class="div-block-14 w-hidden-medium w-hidden-small w-hidden-tiny"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png" width="120" sizes="(max-width: 991px) 100vw, 120px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1600.png 1600w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png 2000w" alt="" class="image-5"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png" width="190" sizes="(max-width: 991px) 100vw, 190px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png 1280w" alt="" class="image-3"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1)-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png 620w" alt="" class="image-4"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png 800w" alt="" class="image-6"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd52f6b273e0001a3d86c_Tableau%20(1).svg" alt="" class="image-7"/></div></div><div class="footer-section"><div class="w-container"><div>© 2024 Metricfire Limited. <br/>All Rights Reserved.<br/><br/><br/>5940 S Rainbow Blvd Ste 400<br/>Las Vegas, NV 89118-2507<br/>United States</div></div></div></div><div class="w-embed"></div><div class="w-embed w-script"><script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Article",
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "https://www.hostedgraphite.com/blog/adventures-in-fault-tolerant-alerting-with-python"
},
"headline": "Adventures in fault tolerant alerting with Python",
"image": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad0843d91f6b21c62e16e15_jcb_flowers.jpg",
"thumbnailUrl": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad0843d91f6b21c62e16e15_jcb_flowers.jpg",
"author": {
"@type": "Person",
"name": "Charlie von Metzradt",
"url": "charlie-von-metzradt"
},
"publisher": {
"@type": "Organization",
"name": "Metricfire Limited",
"logo": {
"@type": "ImageObject",
"url": "https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg"
}
},
"datePublished": "May 05, 2017",
"dateModified": "Sep 29, 2021",
"description": "Learn about Hosted Graphite&#39;s presentation at Python Ireland&#39;s April meetup using PySyncObj. See what type of tech our engineers have chosen and why."
}
</script>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "SoftwareApplication",
"applicationCategory": "BusinessApplication",
"name": "Hosted Graphite",
"description": "Hosted Graphite offers powerful monitoring and alerting for various business applications, providing integrations with multiple data sources, a Grafana UI, and great customer support.",
"review": [
{
"@type": "Review",
"author": { "@type": "Person", "name": "Cecily G." },
"reviewBody": "This system is the tool every engineer needs to be successful. Easy to use and captures metrics that no other system has successfully been able to pull for us.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Christopher C." },
"reviewBody": "Great product suite, outstanding customer service! Running a distributed data pipeline is challenging to keep healthy, and Hosted Graphite has been an instrumental partner for monitoring system health and getting ahead of issues.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "UC" },
"reviewBody": "A great service with excellent support. Simple to use, inexpensive, and great support.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Brendan C." },
"reviewBody": "High-quality metrics hosting. Hosted Graphite enables us to instrument our products and processes efficiently. We can proactively monitor for issues and use dashboards to identify more minor performance problems before they become major ones.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Jacobson M." },
"reviewBody": "Perfect for remote device analytics. Hosted graphite makes it easy for me to keep track of my remote devices being used by my customers. I can't always be on site, so this allows easy tracking of device details.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "James C." },
"reviewBody": "The easiest way for DevOps to incorporate Kubernetes without having to suffer a paradigm shift away from using Graphite. Prometheus, while academically good, forces security worst practices which were unacceptable for us as a SOC2 compliant organization. Graphite is battle-proven and already integrated, so Hosted Graphite allowed us to do what we needed without drastic changes.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
"datePublished": "2021-03-19"
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Anne M." },
"reviewBody": "Offers great hosting service, Hosted Graphite is quite easy to use and highly secure. I like how they offer the trial version to evaluate if the solution fits your needs. In terms of setup, it's easy with minimal registration details and flexible pricing.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
"datePublished": "2019-02-20"
}
],
"operatingSystem": "All major operating systems",
"aggregateRating": {
"@type": "AggregateRating",
"reviewCount": 7,
"ratingValue": 4.85,
"bestRating": 5,
"worstRating": 1
},
"offers": {
"@type": "Offer",
"price": 19.00,
"priceCurrency": "USD",
"availability": "https://schema.org/InStock",
"url": "https://hostedgraphite.com",
"seller": {
"@type": "Organization",
"name": "Hosted Graphite"
}
}
}
</script></div><script src="https://d3e54v103j8qbb.cloudfront.net/js/jquery-3.5.1.min.dc5e7f18c8.js?site=5a57aa76d1fa2300015ca244" type="text/javascript" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.36b8fb49256177c8.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.83de43c5e9eda212.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.54e1a7f0.3d4248ede2db61fe.js" type="text/javascript"></script><!-- Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-NT79GSP"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) --><script>
img = document.querySelectorAll('.w-embed');
img.forEach(element => element.style.textAlign = 'center');
</script></body></html>