Files
nexus/sreweekly/articles/151/01-a-victim-of-its-own-popularity-scaling-our-cloudwatch-integration.html
2026-09-12 17:23:01 +08:00

351 lines
39 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><!-- Last Published: Wed Jul 02 2025 01:43:30 GMT+0000 (Coordinated Universal Time) --><html data-wf-domain="webflow.hostedgraphite.com" data-wf-page="5ce3e821b5d6678199f31df7" data-wf-site="5a57aa76d1fa2300015ca244" data-wf-collection="5ce3e821b5d6674c4bf31e14" data-wf-item-slug="a-victim-of-its-own-popularity-scaling-our-cloudwatch-integration"><head><meta charset="utf-8"/><link href="https://cdn.prod.website-files.com" rel="preconnect" crossorigin="anonymous"/><title>A victim of its own popularity: Scaling our CloudWatch integration</title><meta content="Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production." name="description"/><meta content="A victim of its own popularity: Scaling our CloudWatch integration" property="og:title"/><meta content="Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production." property="og:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png" property="og:image"/><meta content="A victim of its own popularity: Scaling our CloudWatch integration" name="twitter:title"/><meta content="Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production." name="twitter:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png" name="twitter:image"/><meta property="og:type" content="website"/><meta content="summary_large_image" name="twitter:card"/><meta content="width=device-width, initial-scale=1" name="viewport"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/css/hosted-graphite-blog.webflow.shared.f88b646c4.min.css" rel="stylesheet" type="text/css"/><link href="https://fonts.googleapis.com" rel="preconnect"/><link href="https://fonts.gstatic.com" rel="preconnect" crossorigin="anonymous"/><script src="https://ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js" type="text/javascript"></script><script type="text/javascript">WebFont.load({ google: { families: ["Open Sans:300,300italic,400,400italic,600,600italic,700,700italic,800,800italic","Inconsolata:400,700","Montserrat:100,100italic,200,200italic,300,300italic,400,400italic,500,500italic,600,600italic,700,700italic,800,800italic,900,900italic","Lato:100,100italic,300,300italic,400,400italic,700,700italic,900,900italic","Droid Sans:400,700","Vollkorn:400,400italic,700,700italic","Ubuntu:300,300italic,400,400italic,500,500italic,700,700italic","Lora:regular,italic,700","Oxygen:300,regular,700","Source Sans Pro:200,200italic,300,300italic,regular,600,600italic,700,700italic,900,900italic"] }});</script><script type="text/javascript">!function(o,c){var n=c.documentElement,t=" w-mod-";n.className+=t+"js",("ontouchstart"in o||o.DocumentTouch&&c instanceof DocumentTouch)&&(n.className+=t+"touch")}(window,document);</script><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f272bbb5a83b7b3239ad6_32.png" rel="shortcut icon" type="image/x-icon"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f27323f9c40d523c9bfcc_256.png" rel="apple-touch-icon"/><script src="https://cdn.jsdelivr.net/npm/code-prettify@0.1.0/src/prettify.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script type="text/javascript">
var embeddify = function(){
$("h6").each(function(index){
var h6 = $(this);
var url = h6.text();
console.log("Embeddifying " + url);
h6.hide();
var embed_message = $('<pre>Embedding <a href="'+url+'" target="_blank">some code</a>, one moment...</pre>');
embed_message.insertAfter(h6);
$.ajax(url, {
success: function(content){
$('<?prettify linenums=true?><pre class="prettyprint">'+content+'</pre>').insertAfter(h6);
embed_message.hide();
PR.prettyPrint();
},
error: function(jqXHR, textStatus, errorThrown){
embed_message.html('Tried to fetch <a href="'+url+'" target="_blank">some code from GitHub</a>, but it failed with an error: "'+errorThrown+'"');
}
});
});
};
var webflow_removed = false;
var attempts_remaining = 10;
function dewebflow() {
/* Temporarily remove this badge while we're sorting out
serving this from the right domains so the hosting provider
will remove the badge properly. */
var badges = $("a.w-webflow-badge");
if(badges.length == 1)
{
$("a.w-webflow-badge").remove();
webflow_removed = true;
} else {
attempts_remaining--;
}
if(!webflow_removed || attempts_remaining > 0)
setTimeout(dewebflow, 500);
}
$(document).ready(embeddify);
$(document).ready(dewebflow);
</script>
<style type"text/css">
/* Turn on line numbering for every line of prettyprinted code, instead of every fifth line. */
li.L0, li.L1, li.L2, li.L3,
li.L5, li.L6, li.L7, li.L8 {
list-style-type: decimal !important;
}
</style>
<!-- Google Tag Manager -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer','GTM-NT79GSP');</script>
<!-- End Google Tag Manager --></head><body class="body-9"><div data-collapse="medium" data-animation="default" data-duration="200" data-easing="ease" data-easing2="ease" role="banner" class="navbar-9 w-nav"><div class="section-15"><div class="container-12 w-container"><nav role="navigation" class="nav-menu-5 w-clearfix w-nav-menu"><a href="https://www.hostedgraphite.com/accounts/signup/" class="hg-nav-link right marketing get-started-ppc-button w-nav-link">Get started free</a><a href="https://www.hostedgraphite.com/accounts/login/" class="hg-nav-link right w-nav-link">Login</a><a href="https://www.hostedgraphite.com/enterprise" class="hg-nav-link w-nav-link">Enterprise</a><a href="https://www.hostedgraphite.com/pricing" class="hg-nav-link w-nav-link">Pricing</a><a href="https://www.hostedgraphite.com/docs/" class="hg-nav-link w-nav-link">Docs</a><a href="https://www.hostedgraphite.com/customers" class="hg-nav-link w-hidden-medium w-hidden-small w-hidden-tiny w-nav-link">Customers</a><a href="https://www.hostedgraphite.com/product" class="hg-nav-link w-nav-link">Features</a></nav><div class="menu-button-3 w-nav-button"><div class="w-icon-nav-menu"></div></div></div></div><div class="div-block-15"><a href="https://www.hostedgraphite.com" class="brand-4 w-nav-brand"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg" width="1088.5" alt="" class="image-52"/></a></div><div class="section-16"></div></div><div data-w-expand="category" style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png&quot;)" class="hero-blog"></div><div class="main-section"><div class="w-container"><div class="section-heading"><h1 class="blog-post-title">A victim of its own popularity: Scaling our CloudWatch integration</h1><div class="blog-date">December 4, 2018</div><a data-w-expand="category" href="/blog-categories/engineering" class="blog-category">Engineering</a><div class="full-divide"></div><div class="w-embed w-script"><script>
document.addEventListener("DOMContentLoaded", function() {
const tocContainer = document.createElement('div');
tocContainer.id = 'toc-container';
tocContainer.innerHTML = '<h2>Table of Contents</h2><ul id="toc"></ul>';
const sectionHeading = document.querySelector('.section-heading');
sectionHeading.insertAdjacentElement('afterend', tocContainer);
const toc = document.getElementById('toc');
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
headers.forEach((header) => {
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
header.id = id;
const li = document.createElement('li');
const a = document.createElement('a');
a.href = `#${id}`;
a.textContent = header.textContent;
li.appendChild(a);
toc.appendChild(li);
});
document.querySelectorAll('#toc a').forEach(anchor => {
anchor.addEventListener('click', function(e) {
e.preventDefault();
document.querySelector(this.getAttribute('href')).scrollIntoView({
behavior: 'smooth'
});
history.pushState(null, null, this.getAttribute('href'));
});
});
});
</script>
<style>
#toc-container {
text-align: start;
width: 85%;
margin: 0 auto 3em;
}
#toc-container h2 {
margin-top: 0;
}
#toc-container ul {
padding-left: 1em;
}
#toc-container li {
margin-bottom: 0.5em;
}
#toc li a {
text-decoration: none;
color: #2e2e2e;
}
#toc li a:hover {
text-decoration: underline;
}
</style></div></div><div class="w-condition-invisible w-embed"></div><div class="blog-content"><div class="blog-post w-richtext"><p>This is the story of how we scaled out our <a href="https://aws.amazon.com/cloudwatch/">AWS CloudWatch</a> integration and what we learned along the way, in particular about incremental deployment using <a href="https://en.wikipedia.org/wiki/Feature_toggle">feature flags</a>.</p><h1>The trouble with success</h1><p>Once upon a time, a customer asked us if we could help them pull their AWS CloudWatch data into Hosted Graphite, so they could store it long-term alongside other system and application metrics.</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/A+victim+of+its+own+popularity%3A+Scaling+our+CloudWatch+integration/aws1.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/A+victim+of+its+own+popularity%3A+Scaling+our+CloudWatch+integration/aws1.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/A+victim+of+its+own+popularity%3A+Scaling+our+CloudWatch+integration/aws1.webp" alt="A diagram showing how metrics were pulled to AWS">
</picture></div><p>‍</p><p>We put together a simple <a href="https://www.hostedgraphite.com/docs/integrationguide/ig_aws_cloudwatch.html">integration</a> comprising an allocator, a job queue, and a worker pool using the <a href="http://python-rq.org/">Redis Queue (RQ)</a> library. A job fetched all metrics configured for a single access key in a given time frame, typically 10 minutes.<br/><br/>This, plus a nice configuration UI, got the job done with minimal setup and development. Customers loved it! We expanded the range of AWS services it could pull data for. Over time, the integration’s popularity grew and grew—and so did the resources required to run it.</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/A+victim+of+its+own+popularity%3A+Scaling+our+CloudWatch+integration/cpupanel2.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/A+victim+of+its+own+popularity%3A+Scaling+our+CloudWatch+integration/cpupanel2.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/A+victim+of+its+own+popularity%3A+Scaling+our+CloudWatch+integration/cpupanel2.webp" alt="A screenshot of a Hosted Graphite dashboard panel monitoring User CPU percentage">
</picture></div><p><br/>This could result in slower throughput, and delays fetching metrics for customers. Arguably a nice problem to have: the initial, simple solution was outgrowing its (single) machine resources.</p><p>We maintain a fairly homogeneous cohort of bare-metal servers, and try to avoid <a href="https://en.wikipedia.org/wiki/Scalability#Horizontal_and_vertical_scaling">scaling vertically</a> wherever we can. Our SRE team have Understandably Strong Feelings about single-homed services, too: while it was possible to catch up on CloudWatch data when we failed over to another machine, it required manual work from both the SRE and development teams. No-one should ever have to wake up to do something a computer could.</p><p>So this was a good opportunity both to split the work over multiple machines, and to make a more reliable service with automated failover.</p><h1>What to do …</h1><p>We needed three things:</p><ol role="list"><li>To split the work across a (changing) number of machines;</li><li>No impact should one or two of those machines fall over; and</li><li>As far as <a href="https://bravenewgeek.com/you-cannot-have-exactly-once-delivery/">possible</a>, “just once” jobs—re-running jobs can land customers with larger AWS bills, as well as ingesting duplicate data (visible under certain of our <a href="https://www.hostedgraphite.com/blog/advanced-data-views-better-observability-more-control">data views</a>).</li></ol><p>Alongside these, we wanted to hold our <a href="https://landing.google.com/sre/sre-book/chapters/service-level-objectives/">service level objectives</a> invariant. Since we define these <a href="https://landing.google.com/sre/sre-book/chapters/monitoring-distributed-systems/">in terms of customer impact</a>, they didn’t need to change as we scaled out, except that they became easier to meet. Without going into boring detail, we had SLOs for job delay, number of attempted jobs, and number of successful jobs.</p><p>Our initial design–allocator, job queue, worker pool–worked well, and was chosen partly because it should ease horizontal scaling. So the plan was to replicate that setup across a few machines, elect an allocating leader, and distribute jobs across all workers in the cluster.</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/A+victim+of+its+own+popularity%3A+Scaling+our+CloudWatch+integration/aws3.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/A+victim+of+its+own+popularity%3A+Scaling+our+CloudWatch+integration/aws3.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/A+victim+of+its+own+popularity%3A+Scaling+our+CloudWatch+integration/aws3.webp" alt="A diagram of how metrics are being pulled into AWS">
</picture></div><p>‍</p><h1>… and how to do it</h1><p>Simple enough, but the devil is as usual in the details. How do we determine the leader? How does the leader know which machines to allocate jobs to? What happens if a machine goes down? What happens if the leader goes down?</p><p>First up, we needed to know what machines were members of the cluster, and their health status. This is the classic <a href="https://en.wikipedia.org/wiki/Service_discovery">service discovery</a> problem, and it was easy for us to base a solution on <a href="https://coreos.com/etcd/">etcd</a>—we already used it for just this purpose. Each machine writes a short-TTL key into etcd to announce that it’s present and healthy.</p><p>We also needed simple <a href="https://en.wikipedia.org/wiki/Leader_election">leader election</a>, so a single allocator could coordinate the cluster. Each machine periodically tries to acquire an etcd <a href="https://coreos.com/etcd/docs/latest/learning/api.html">lock</a>: the successful machine is the current leader.</p><p>Now our leader is ready to allocate jobs! It maintains an RQ queue on each machine in the cluster, and assigns jobs by hashing over the set of healthy machines. It also maintains a mapping—shared across the whole cluster—of what jobs are allocated where, so that in the event of leader failover, we can easily figure out what jobs need to be reallocated.</p><p>We modified the behaviour of the job slightly, too: it reports completion times to every machine in the cluster. Then after a machine failure—leader or otherwise—it’s easy to see when a job was last completed and reschedule it, replaying any missing data from missed runs.</p><h1>Getting to production</h1><p>That all sounds good, but it was a big shift in how our CloudWatch service operated: how could we do it without impacting our customers? Rather than choosing a <a href="https://en.wikipedia.org/wiki/Flag_day_(computing)">flag day</a> to cut over to a new solution, we went with an incremental approach—each key feature was rolled out gradually and in a safe, controlled way.</p><p>Deploying the service discovery part and verifying it was OK was easy: just making sure the right stuff ended up in etcd.</p><p>To gain confidence in the leader election part, we ran machines alongside the “real” service and deployed “fake” election capabilities: all machines would vie to acquire the lock, but actual leader behaviour was tucked behind a <a href="https://en.wikipedia.org/wiki/Feature_toggle">feature flag</a>.</p><p>Our next deployment synced leader context to followers. Next came “fake” job hashing, and then creation of RQ queues across the cluster. Finally, we deployed a mechanism to write back job results.</p><p>At this stage we had all of the important components working in the background successfully, while at the same time our original machine operated as normal.</p><p>All that remained was to flip our feature flag such that leader election &amp; job distribution came into effect. So we flipped it and … it worked! What seemed like a potentially daunting deployment process ended up being quite easy.</p><p>Although this incremental approach added some development overhead, it was entirely worth it: we had no outages or customer impact at all during the rollout.</p><h1>The steady state</h1><p>We can now expand our CloudWatch integration—either in the number of AWS services we support, or the number of customers using it—without worrying about capacity. Growing is easy: we just provision a new machine for the cluster and it automatically joins and starts working.</p><p>We’ve watched (and intentionally triggered!) various leader failovers, but only by looking at graphs: it happens as seamlessly and automatically as we’d hoped.</p><p>‍</p></div><div class="sticky-toc-container"><div class="cta-sidebar"><div class="text-block-31"><strong class="bold-text-15">Try Hosted Graphite now!</strong></div><p class="paragraph-12">Get Hosted Graphite free for 14 days. No credit card required.</p><a href="https://www.hostedgraphite.com/accounts/signup/?from=heroku_blogpost" class="cta-side-button w-button">Get Started</a></div><div class="toc w-embed w-script"><script>
document.addEventListener("DOMContentLoaded", function() {
function createSideToc(containerId) {
const container = document.querySelector(containerId);
// Create and append the title <h2>
const tocTitle = document.createElement('h2');
tocTitle.textContent = 'Table of Contents';
container.appendChild(tocTitle);
const toc = document.createElement('ul');
toc.className = 'toc';
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
const ids = new Set(); // To ensure unique IDs
headers.forEach((header) => {
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
header.id = id;
if (!ids.has(id)) {
ids.add(id);
const li = document.createElement('li');
const a = document.createElement('a');
a.href = `#${id}`;
a.textContent = header.textContent;
li.appendChild(a);
toc.appendChild(li);
}
});
container.appendChild(toc);
// Smooth scroll for the new TOC
toc.querySelectorAll('a').forEach(anchor => {
anchor.addEventListener('click', function(e) {
e.preventDefault();
document.querySelector(this.getAttribute('href')).scrollIntoView({
behavior: 'smooth'
});
history.pushState(null, null, this.getAttribute('href'));
});
});
}
// Create the sticky TOC in the sidebar
createSideToc('.sticky-toc-container');
});
</script>
<style>
/* Specific Styling for Sticky TOC */
.sticky-toc-container {
position: sticky;
top: 110px;
width: 340px;
height: 100%;
overflow-y: auto;
padding: 15px;
border-radius: 8px;
font-family: Arial, sans-serif;
font-size: 14px;
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
text-align: left;
}
.sticky-toc-container h2 {
margin-top: 0;
font-size: 18px;
color: #2e2e2e;
}
.sticky-toc-container ul {
list-style-type: none;
padding-left: 0;
margin: 0;
}
.sticky-toc-container li {
margin-bottom: 0.6em;
}
.sticky-toc-container a {
text-decoration: none;
color: #8e8b8b;
}
.sticky-toc-container a:hover {
text-decoration: underline;
}
/* Make sure the TOC has a scroll bar when needed */
.toc {
max-height: 35vh;
overflow-y: auto;
}
/* Media Query to Hide sticky-toc-container on smaller screens */
@media (max-width: 1215px) {
.sticky-toc-container {
display: none;
}
}
</style></div></div></div><div class="w-condition-invisible w-embed w-script"><script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js"></script>
<pre class="prettyprint">
</pre></div><div class="div-block-2"><div class="full-divide"></div><div class="author-wrapper"><a href="/blog-authors/ciaran-egan" data-w-expand="authors" class="author-name">Ciaran Egan</a><div class="smallest-divider"></div><div data-w-expand="authors" class="author-bio w-richtext"><p>Developer at Hosted Graphite.</p></div><div><a href="https://twitter.com/hostedgraphite" data-w-expand="authors" class="social-link w-inline-block"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a57aa7ad1fa2300015ca29a_social-18.svg" width="68" alt="" class="image-2"/></a><a data-w-expand="authors" href="#" class="social-link w-inline-block"></a><a href="#" data-w-expand="authors" class="social-link w-inline-block"></a></div></div></div></div></div><div class="main-section gray"><div class="w-container"><div class="section-heading"><h2 class="heading-6">Related Posts</h2><div class="med-divider"></div></div><div class="w-dyn-list"><div role="list" class="w-clearfix w-dyn-items w-row"><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/hosted-graphite-isnt-graphite" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5dd37e9fe192cd2ea5b8af68_Screen%20Shot%202019-11-19%20at%202.14.02%20PM.jpg&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">HostedGraphite</div><div class="preview-text">Hosted Graphite improves upon standard Graphite. Take a look at how we do this and how Hosted Graphite gives your company better functionality. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Shevaun Frazier</div><div class="thumbnail-date">Nov 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/new-developer-onboarding" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5bfea01e7ffa1b30f88f19c0_Blog-background-organge.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">New Developer Onboarding</div><div class="preview-text">Discussions about onboarding tend to revolve around new hires, for obvious reasons, but the process is important for everyone: while the new developer learns the most important aspects of team and company culture, the team has an opportunity to learn new ideas from a fresh pair of eyes. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Heather Wiencko</div><div class="thumbnail-date">Sep 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/pug-life-how-we-run-a-grafana-instance-for-each-user-with-docker" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5d643d90b1c59597a144484a_pug-header.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">PUG Life: How we run a Grafana instance for each user with Docker </div><div class="preview-text">The story of Hosted Grafana at Hosted Graphite, and how we run multiple instances of Grafana. Start monitoring with Hosted Graphite and use our Grafana dashboards directly in-platform. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciarán Finn</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/incident-postmortem-template" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Our incident postmortem template</div><div class="preview-text">Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Aug 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/its-dead-jim-how-we-write-an-incident-postmortem" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">&quot;It&#x27;s dead, Jim&quot;: How we write an incident postmortem</div><div class="preview-text">How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Jul 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/surviving-on-call-tips-from-a-hosted-graphite-sre" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c48a4405b91aaf448087677_daveblog-header.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Surviving On-Call: Tips from a Hosted Graphite SRE</div><div class="preview-text">Get an insight into what Hosted Graphite&#x27;s SRE Dave Fennell has learned when being On-Call. Read the tips that he has to offer to make the On-Call experience a positive one.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Dave Fennell</div><div class="thumbnail-date">Jan 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/deadlines-lies-and-videotape-the-tale-of-a-grpc-bug" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Deadlines lies and videotape: The tale of a gRPC bug</div><div class="preview-text">If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released).</div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Gaffney</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/status-page-updates-its-all-about-timing" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Status page updates: It’s all about timing</div><div class="preview-text">Check out part 2 of Hosted Graphite&#x27;s SRE process on how to handing status page updates. Learn more in detail on the responsibilities of communication, duration of updating the page, and who’s responsible for certain tasks.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Oct 2018</div></div></a></div></div></div></div></div><div class="main-section dark"><div class="container-13 w-container"><div class="section-heading"><h2 class="white">See why thousands of engineers trust Hosted Graphite with their monitoring</h2><div class="med-divider"></div></div><a href="https://www.hostedgraphite.com/accounts/signup/" class="get-started-ppc-button w-button">START A FREE TRIAL</a><div class="div-block-14 w-hidden-medium w-hidden-small w-hidden-tiny"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png" width="120" sizes="(max-width: 991px) 100vw, 120px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1600.png 1600w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png 2000w" alt="" class="image-5"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png" width="190" sizes="(max-width: 991px) 100vw, 190px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png 1280w" alt="" class="image-3"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1)-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png 620w" alt="" class="image-4"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png 800w" alt="" class="image-6"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd52f6b273e0001a3d86c_Tableau%20(1).svg" alt="" class="image-7"/></div></div><div class="footer-section"><div class="w-container"><div>© 2024 Metricfire Limited. <br/>All Rights Reserved.<br/><br/><br/>5940 S Rainbow Blvd Ste 400<br/>Las Vegas, NV 89118-2507<br/>United States</div></div></div></div><div class="w-embed"></div><div class="w-embed w-script"><script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Article",
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "https://www.hostedgraphite.com/blog/a-victim-of-its-own-popularity-scaling-our-cloudwatch-integration"
},
"headline": "A victim of its own popularity: Scaling our CloudWatch integration",
"image": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png",
"thumbnailUrl": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png",
"author": {
"@type": "Person",
"name": "Ciaran Egan",
"url": "ciaran-egan"
},
"publisher": {
"@type": "Organization",
"name": "Metricfire Limited",
"logo": {
"@type": "ImageObject",
"url": "https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg"
}
},
"datePublished": "Dec 04, 2018",
"dateModified": "Sep 22, 2021",
"description": "Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production. "
}
</script>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "SoftwareApplication",
"applicationCategory": "BusinessApplication",
"name": "Hosted Graphite",
"description": "Hosted Graphite offers powerful monitoring and alerting for various business applications, providing integrations with multiple data sources, a Grafana UI, and great customer support.",
"review": [
{
"@type": "Review",
"author": { "@type": "Person", "name": "Cecily G." },
"reviewBody": "This system is the tool every engineer needs to be successful. Easy to use and captures metrics that no other system has successfully been able to pull for us.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Christopher C." },
"reviewBody": "Great product suite, outstanding customer service! Running a distributed data pipeline is challenging to keep healthy, and Hosted Graphite has been an instrumental partner for monitoring system health and getting ahead of issues.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "UC" },
"reviewBody": "A great service with excellent support. Simple to use, inexpensive, and great support.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Brendan C." },
"reviewBody": "High-quality metrics hosting. Hosted Graphite enables us to instrument our products and processes efficiently. We can proactively monitor for issues and use dashboards to identify more minor performance problems before they become major ones.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Jacobson M." },
"reviewBody": "Perfect for remote device analytics. Hosted graphite makes it easy for me to keep track of my remote devices being used by my customers. I can't always be on site, so this allows easy tracking of device details.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "James C." },
"reviewBody": "The easiest way for DevOps to incorporate Kubernetes without having to suffer a paradigm shift away from using Graphite. Prometheus, while academically good, forces security worst practices which were unacceptable for us as a SOC2 compliant organization. Graphite is battle-proven and already integrated, so Hosted Graphite allowed us to do what we needed without drastic changes.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
"datePublished": "2021-03-19"
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Anne M." },
"reviewBody": "Offers great hosting service, Hosted Graphite is quite easy to use and highly secure. I like how they offer the trial version to evaluate if the solution fits your needs. In terms of setup, it's easy with minimal registration details and flexible pricing.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
"datePublished": "2019-02-20"
}
],
"operatingSystem": "All major operating systems",
"aggregateRating": {
"@type": "AggregateRating",
"reviewCount": 7,
"ratingValue": 4.85,
"bestRating": 5,
"worstRating": 1
},
"offers": {
"@type": "Offer",
"price": 19.00,
"priceCurrency": "USD",
"availability": "https://schema.org/InStock",
"url": "https://hostedgraphite.com",
"seller": {
"@type": "Organization",
"name": "Hosted Graphite"
}
}
}
</script></div><script src="https://d3e54v103j8qbb.cloudfront.net/js/jquery-3.5.1.min.dc5e7f18c8.js?site=5a57aa76d1fa2300015ca244" type="text/javascript" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.36b8fb49256177c8.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.83de43c5e9eda212.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.54e1a7f0.3d4248ede2db61fe.js" type="text/javascript"></script><!-- Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-NT79GSP"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) --><script>
img = document.querySelectorAll('.w-embed');
img.forEach(element => element.style.textAlign = 'center');
</script></body></html>