Files
nexus/sreweekly/articles/86/09-per-metric-rate-limiting-how-we-protect-our-backend.html
2026-09-12 17:23:01 +08:00

355 lines
41 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><!-- Last Published: Wed Jul 02 2025 01:43:30 GMT+0000 (Coordinated Universal Time) --><html data-wf-domain="webflow.hostedgraphite.com" data-wf-page="5ce3e821b5d6678199f31df7" data-wf-site="5a57aa76d1fa2300015ca244" data-wf-collection="5ce3e821b5d6674c4bf31e14" data-wf-item-slug="per-metric-rate-limiting-how-we-protect-our-backend"><head><meta charset="utf-8"/><link href="https://cdn.prod.website-files.com" rel="preconnect" crossorigin="anonymous"/><title>Per-metric rate limiting: How we protect our backend</title><meta content="Learn how Hosted Graphite limits the amount of data received to protect the backend. Learn how the per-metric rate-limiting was built from the ground up." name="description"/><meta content="Per-metric rate limiting: How we protect our backend" property="og:title"/><meta content="Learn how Hosted Graphite limits the amount of data received to protect the backend. Learn how the per-metric rate-limiting was built from the ground up." property="og:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad096b41d8156956a76d15b_screenshot_2017-08-09_16-51-29.png" property="og:image"/><meta content="Per-metric rate limiting: How we protect our backend" name="twitter:title"/><meta content="Learn how Hosted Graphite limits the amount of data received to protect the backend. Learn how the per-metric rate-limiting was built from the ground up." name="twitter:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad096b41d8156956a76d15b_screenshot_2017-08-09_16-51-29.png" name="twitter:image"/><meta property="og:type" content="website"/><meta content="summary_large_image" name="twitter:card"/><meta content="width=device-width, initial-scale=1" name="viewport"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/css/hosted-graphite-blog.webflow.shared.f88b646c4.min.css" rel="stylesheet" type="text/css"/><link href="https://fonts.googleapis.com" rel="preconnect"/><link href="https://fonts.gstatic.com" rel="preconnect" crossorigin="anonymous"/><script src="https://ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js" type="text/javascript"></script><script type="text/javascript">WebFont.load({ google: { families: ["Open Sans:300,300italic,400,400italic,600,600italic,700,700italic,800,800italic","Inconsolata:400,700","Montserrat:100,100italic,200,200italic,300,300italic,400,400italic,500,500italic,600,600italic,700,700italic,800,800italic,900,900italic","Lato:100,100italic,300,300italic,400,400italic,700,700italic,900,900italic","Droid Sans:400,700","Vollkorn:400,400italic,700,700italic","Ubuntu:300,300italic,400,400italic,500,500italic,700,700italic","Lora:regular,italic,700","Oxygen:300,regular,700","Source Sans Pro:200,200italic,300,300italic,regular,600,600italic,700,700italic,900,900italic"] }});</script><script type="text/javascript">!function(o,c){var n=c.documentElement,t=" w-mod-";n.className+=t+"js",("ontouchstart"in o||o.DocumentTouch&&c instanceof DocumentTouch)&&(n.className+=t+"touch")}(window,document);</script><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f272bbb5a83b7b3239ad6_32.png" rel="shortcut icon" type="image/x-icon"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f27323f9c40d523c9bfcc_256.png" rel="apple-touch-icon"/><script src="https://cdn.jsdelivr.net/npm/code-prettify@0.1.0/src/prettify.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script type="text/javascript">
var embeddify = function(){
$("h6").each(function(index){
var h6 = $(this);
var url = h6.text();
console.log("Embeddifying " + url);
h6.hide();
var embed_message = $('<pre>Embedding <a href="'+url+'" target="_blank">some code</a>, one moment...</pre>');
embed_message.insertAfter(h6);
$.ajax(url, {
success: function(content){
$('<?prettify linenums=true?><pre class="prettyprint">'+content+'</pre>').insertAfter(h6);
embed_message.hide();
PR.prettyPrint();
},
error: function(jqXHR, textStatus, errorThrown){
embed_message.html('Tried to fetch <a href="'+url+'" target="_blank">some code from GitHub</a>, but it failed with an error: "'+errorThrown+'"');
}
});
});
};
var webflow_removed = false;
var attempts_remaining = 10;
function dewebflow() {
/* Temporarily remove this badge while we're sorting out
serving this from the right domains so the hosting provider
will remove the badge properly. */
var badges = $("a.w-webflow-badge");
if(badges.length == 1)
{
$("a.w-webflow-badge").remove();
webflow_removed = true;
} else {
attempts_remaining--;
}
if(!webflow_removed || attempts_remaining > 0)
setTimeout(dewebflow, 500);
}
$(document).ready(embeddify);
$(document).ready(dewebflow);
</script>
<style type"text/css">
/* Turn on line numbering for every line of prettyprinted code, instead of every fifth line. */
li.L0, li.L1, li.L2, li.L3,
li.L5, li.L6, li.L7, li.L8 {
list-style-type: decimal !important;
}
</style>
<!-- Google Tag Manager -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer','GTM-NT79GSP');</script>
<!-- End Google Tag Manager --></head><body class="body-9"><div data-collapse="medium" data-animation="default" data-duration="200" data-easing="ease" data-easing2="ease" role="banner" class="navbar-9 w-nav"><div class="section-15"><div class="container-12 w-container"><nav role="navigation" class="nav-menu-5 w-clearfix w-nav-menu"><a href="https://www.hostedgraphite.com/accounts/signup/" class="hg-nav-link right marketing get-started-ppc-button w-nav-link">Get started free</a><a href="https://www.hostedgraphite.com/accounts/login/" class="hg-nav-link right w-nav-link">Login</a><a href="https://www.hostedgraphite.com/enterprise" class="hg-nav-link w-nav-link">Enterprise</a><a href="https://www.hostedgraphite.com/pricing" class="hg-nav-link w-nav-link">Pricing</a><a href="https://www.hostedgraphite.com/docs/" class="hg-nav-link w-nav-link">Docs</a><a href="https://www.hostedgraphite.com/customers" class="hg-nav-link w-hidden-medium w-hidden-small w-hidden-tiny w-nav-link">Customers</a><a href="https://www.hostedgraphite.com/product" class="hg-nav-link w-nav-link">Features</a></nav><div class="menu-button-3 w-nav-button"><div class="w-icon-nav-menu"></div></div></div></div><div class="div-block-15"><a href="https://www.hostedgraphite.com" class="brand-4 w-nav-brand"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg" width="1088.5" alt="" class="image-52"/></a></div><div class="section-16"></div></div><div data-w-expand="category" style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad096b41d8156956a76d15b_screenshot_2017-08-09_16-51-29.png&quot;)" class="hero-blog"></div><div class="main-section"><div class="w-container"><div class="section-heading"><h1 class="blog-post-title">Per-metric rate limiting: How we protect our backend</h1><div class="blog-date">August 15, 2017</div><a data-w-expand="category" href="/blog-categories/engineering" class="blog-category">Engineering</a><div class="full-divide"></div><div class="w-embed w-script"><script>
document.addEventListener("DOMContentLoaded", function() {
const tocContainer = document.createElement('div');
tocContainer.id = 'toc-container';
tocContainer.innerHTML = '<h2>Table of Contents</h2><ul id="toc"></ul>';
const sectionHeading = document.querySelector('.section-heading');
sectionHeading.insertAdjacentElement('afterend', tocContainer);
const toc = document.getElementById('toc');
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
headers.forEach((header) => {
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
header.id = id;
const li = document.createElement('li');
const a = document.createElement('a');
a.href = `#${id}`;
a.textContent = header.textContent;
li.appendChild(a);
toc.appendChild(li);
});
document.querySelectorAll('#toc a').forEach(anchor => {
anchor.addEventListener('click', function(e) {
e.preventDefault();
document.querySelector(this.getAttribute('href')).scrollIntoView({
behavior: 'smooth'
});
history.pushState(null, null, this.getAttribute('href'));
});
});
});
</script>
<style>
#toc-container {
text-align: start;
width: 85%;
margin: 0 auto 3em;
}
#toc-container h2 {
margin-top: 0;
}
#toc-container ul {
padding-left: 1em;
}
#toc-container li {
margin-bottom: 0.5em;
}
#toc li a {
text-decoration: none;
color: #2e2e2e;
}
#toc li a:hover {
text-decoration: underline;
}
</style></div></div><div class="w-condition-invisible w-embed"></div><div class="blog-content"><div class="blog-post w-richtext"><p>To protect our backend and make sure one customer&#x27;s traffic can&#x27;t affect anyone else, we carefully limit the amount of data we accept from our users. This data can arrive at different rates and is immediately limited if we suspect it to be harmful. However, there are some cases where traffic will pass by our initial checks but still cause problems further down the data ingestion pipeline. One case we solved recently is when a user is sending a high volume of datapoints per second to a single metric. That’s why we developed per-metric rate limiting, a system we built from the ground up to limit damaging traffic that’s not intended to harm. </p><h3><strong>The problem</strong></h3><p>If servers receive data at a higher rate than they can process, it can lead to an exhaustion of resources, a backlog of requests and, at worst, data loss. To alleviate the problem, it’s usually possible to follow the traditional approach to rate limiting: applying backpressure to tell the other side to slow down. This lets you control data streams so your server only receives data as fast as it can be processed. </p><p>Backpressure wasn’t an option for us, however. Each individual user and metric does not have their own network connection over which we can apply backpressure. As a result, everything is mixed up into batches, and connections are shared by all user data. The only point at which our pipeline has enough data to know we’re in an over-limit condition for a single metric name is too late in the process to take action. </p><p>We realised we needed a totally new approach, one that would allow the traffic to flow normally and use the over-limit information late in the process to change the behaviour of earlier parts of the pipeline. In addition, our rate limiting system needed to react to changing levels in traffic (knowing that it’s causing some of the changes itself), be consistent across machines (so that multiple servers could access the same information and react to it) and act relatively fast (for speedy reactions to changing traffic).</p><h3><strong>Two layers of traffic ingestion</strong></h3><p>Our backend deals with two layers of traffic ingestion: the first, <strong>the load balancing layer, </strong>forwards to a second, <strong>the aggregation layer</strong>. However, no communication was previously flowing from the aggregation back to the load balancing layer. The traffic being forwarded from the load balancing to the aggregation layer takes the following form:</p><p><em>&lt;metric_name&gt; &lt;value&gt; &lt;timestamp&gt;</em>‍</p><p><em>my.metric.one.count 23      1501519326     </em> </p><h3><strong>The load balancing layer</strong></h3><p>The load balancing layer is optimised for availability and receives metric traffic over all supported protocols. However, there&#x27;s a drawback to optimising for availability (and speed): there’s not enough time to ask all the other endpoints in this layer how much traffic they&#x27;ve seen, and specifically for which metric names. That&#x27;d be a ridiculous and impractical amount of data to sync around. We therefore can’t decide if we should allow the datapoint we&#x27;ve just received progress onto the second layer for data aggregation. After authentication we forward each datapoint to a specific machine in the aggregation layer, where data aggregation happens. Yet, at this layer, each machine only knows what traffic it has seen for each metric. As a result, it’s not possible to make any decisions about rate limiting for a user’s traffic across the whole set of machines in this layer either - there&#x27;s still too much usage data to sync around to make it practical.</p><h3><strong>The aggregation layer</strong></h3><p>In the aggregation layer, we receive data from the load balancing layer and aggregate metrics together into queryable and useful formats. Only here do we have any oversight over how much traffic/how many datapoints each individual metric is receiving. That means that if the rate of datapoints per second is greater than the upper limit of this layer’s aggregation process, the effectiveness of the aggregation layer is compromised, and a user is breaching their account limits without us knowing, and other users can start seeing some impact. Simply dropping the traffic at this level doesn&#x27;t solve the problem as we&#x27;ve already done almost all of the work involved in processing it - it just wouldn&#x27;t help to drop it at this point. However, at this point we do at last have the information about which metrics are seeing too much data, which leads us to...</p><h3><strong>The solution</strong></h3><p>We took several steps to solve this problem. To begin with, we needed to make separate changes to the aggregation and load balancing layers. In the aggregation layer, we set up a check for offending metrics and from that work out what percentage of their traffic needs to be dropped. In the load balancing layer, we looked at what percentage of traffic would need to be dropped for which metrics. We then needed a way of communicating this information from the aggregation layer back to the load balancing layer, where it could be acted on.</p><p>That’s where <a href="https://coreos.com/etcd/">etcd</a> came in, a distributed, key-value store for data of a distributed system. It’s one way to communicate between the two layers and we chose it primarily for its reliability (it’s distributed using Raft), ease of use and speed (<a href="https://github.com/coreos/etcd">benchmarked 10,000 writes/sec</a>). There were several other reasons etcd made sense for us: firstly, we already had it installed and running in our stack. This allowed us to easily test out a proof of concept without needing extra hardware or installing new tech onto our production machines. Secondly, etcd makes security and authentication more manageable: the trees storing metric information can be locked, which offers us another layer of security around potentially sensitive user data. There&#x27;s also a useful <a href="https://github.com/jplana/python-etcd">python-etcd client available</a> which sits nicely into our tech stack (<a href="https://www.hostedgraphite.com/blog/adventures-in-fault-tolerant-alerting-with-python">most of the services that run Hosted Graphite are built in python</a>) and we only needed to store some very simple data (name and value).</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/per-metric-rate-limiting-how-we-protect-our-backend/rate2.webp" type="image/webp">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/per-metric-rate-limiting-how-we-protect-our-backend/rate2.webp" alt="Diagram showing how metrics are being ratelimited in the backend of Hosted Graphite">
</picture></div><p>‍</p><p>With etcd in place, we could tell the load balancing layer what percentage of traffic to drop for a specific metric. The aggregation layer could then calculate what percentage of traffic to drop at each ingestion point in the load balancing layer (in order to bring a metric&#x27;s traffic/datapoint rate below, say, 1,000 per second, or whatever the configured limit is). The load balancing layer could then periodically sync this set of metric names and maintain a lookup of those it needs to start dropping traffic for. If the process in the load balancing layer sees one of these metrics, it begins counting how many data points have been seen for that metric recently. Now that a load balancing process knows what proportion of the traffic it has seen itself, it’s in a position to consider dropping a portion of that traffic in order to reduce the overall traffic level seen at the aggregation layer.</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/per-metric-rate-limiting-how-we-protect-our-backend/rate1.webp" type="image/webp">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/per-metric-rate-limiting-how-we-protect-our-backend/rate1.webp" alt="A sequence diagram showing on how metrics are being sent through different layers in Hosted Graphite">
</picture></div><p>‍</p><p>However, a naive implementation of this creates another problem. As the traffic to the aggregation layer, from a metric which is being ratelimited, drops in response to the limiting undertaken at the load balancing layer, it looks like no more limiting is required and so reacts by not dropping any traffic. The problem, though, is that the user hasn&#x27;t changed the rate they&#x27;re sending at, and we end up in an oscillating situation where we&#x27;re limiting, then not limiting, then limiting, etc. This is bad for the health of the aggregation service because it doesn&#x27;t solve the overload condition, and bad for the user experience because the limiting is unpredictable. The oscillating nature of this traffic actually makes things worse in the aggregation layer because rapidly &quot;banging&quot; traffic volume like this causes instability itself. Smoothing things out should be the goal.</p><p>So, using etcd&#x27;s key TTL/expiry feature and some gradual backoff to recalculate the limit periodically, the limit is dynamically adjusted and will slowly return to a non-limiting state. If the user continues to send at the same rate, the limiting will &quot;wobble&quot; a little, but will stay mostly constant. When the user&#x27;s traffic pattern changes, the limiting will react within a few seconds and either limit more, less, or not at all, as appropriate, and will do so in a controlled way.</p><h3>Results</h3><p>Through per-metric ratelimiting we’ve successfully dropped billions of datapoints that were arriving at dangerous levels, protecting our backend and providing a better experience for our users.</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/per-metric-rate-limiting-how-we-protect-our-backend/hg3.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/per-metric-rate-limiting-how-we-protect-our-backend/hg3.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/per-metric-rate-limiting-how-we-protect-our-backend/hg3.webp" alt="A screenshot of a Hosted Graphite dashboard monitoring "Keys added to etcd by the aggregation layer"">
<figcaption>Keys added to etcd by the aggregation layer over a month</figcaption>
</picture></div><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/per-metric-rate-limiting-how-we-protect-our-backend/hg4.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/per-metric-rate-limiting-how-we-protect-our-backend/hg4.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/per-metric-rate-limiting-how-we-protect-our-backend/hg4.webp" alt="A screenshot of a Hosted Graphite dashboard monitoring datapoints droped due to per metric ratelimiting">
<figcaption>Total datapoints dropped since we began</figcaption>
</picture></div></div><div class="sticky-toc-container"><div class="cta-sidebar"><div class="text-block-31"><strong class="bold-text-15">Try Hosted Graphite now!</strong></div><p class="paragraph-12">Get Hosted Graphite free for 14 days. No credit card required.</p><a href="https://www.hostedgraphite.com/accounts/signup/?from=heroku_blogpost" class="cta-side-button w-button">Get Started</a></div><div class="toc w-embed w-script"><script>
document.addEventListener("DOMContentLoaded", function() {
function createSideToc(containerId) {
const container = document.querySelector(containerId);
// Create and append the title <h2>
const tocTitle = document.createElement('h2');
tocTitle.textContent = 'Table of Contents';
container.appendChild(tocTitle);
const toc = document.createElement('ul');
toc.className = 'toc';
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
const ids = new Set(); // To ensure unique IDs
headers.forEach((header) => {
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
header.id = id;
if (!ids.has(id)) {
ids.add(id);
const li = document.createElement('li');
const a = document.createElement('a');
a.href = `#${id}`;
a.textContent = header.textContent;
li.appendChild(a);
toc.appendChild(li);
}
});
container.appendChild(toc);
// Smooth scroll for the new TOC
toc.querySelectorAll('a').forEach(anchor => {
anchor.addEventListener('click', function(e) {
e.preventDefault();
document.querySelector(this.getAttribute('href')).scrollIntoView({
behavior: 'smooth'
});
history.pushState(null, null, this.getAttribute('href'));
});
});
}
// Create the sticky TOC in the sidebar
createSideToc('.sticky-toc-container');
});
</script>
<style>
/* Specific Styling for Sticky TOC */
.sticky-toc-container {
position: sticky;
top: 110px;
width: 340px;
height: 100%;
overflow-y: auto;
padding: 15px;
border-radius: 8px;
font-family: Arial, sans-serif;
font-size: 14px;
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
text-align: left;
}
.sticky-toc-container h2 {
margin-top: 0;
font-size: 18px;
color: #2e2e2e;
}
.sticky-toc-container ul {
list-style-type: none;
padding-left: 0;
margin: 0;
}
.sticky-toc-container li {
margin-bottom: 0.6em;
}
.sticky-toc-container a {
text-decoration: none;
color: #8e8b8b;
}
.sticky-toc-container a:hover {
text-decoration: underline;
}
/* Make sure the TOC has a scroll bar when needed */
.toc {
max-height: 35vh;
overflow-y: auto;
}
/* Media Query to Hide sticky-toc-container on smaller screens */
@media (max-width: 1215px) {
.sticky-toc-container {
display: none;
}
}
</style></div></div></div><div class="w-condition-invisible w-embed w-script"><script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js"></script>
<pre class="prettyprint">
</pre></div><div class="div-block-2"><div class="full-divide"></div><div class="author-wrapper"><a href="/blog-authors/ciaran-finn" data-w-expand="authors" class="author-name">Ciarán Finn</a><div class="smallest-divider"></div><div data-w-expand="authors" class="author-bio w-richtext"><p>Technical Lead at Hosted Graphite.</p></div><div><a href="https://twitter.com/hostedgraphite" data-w-expand="authors" class="social-link w-inline-block"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a57aa7ad1fa2300015ca29a_social-18.svg" width="68" alt="" class="image-2"/></a><a data-w-expand="authors" href="#" class="social-link w-inline-block"></a><a href="#" data-w-expand="authors" class="social-link w-inline-block"></a></div></div></div></div></div><div class="main-section gray"><div class="w-container"><div class="section-heading"><h2 class="heading-6">Related Posts</h2><div class="med-divider"></div></div><div class="w-dyn-list"><div role="list" class="w-clearfix w-dyn-items w-row"><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/hosted-graphite-isnt-graphite" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5dd37e9fe192cd2ea5b8af68_Screen%20Shot%202019-11-19%20at%202.14.02%20PM.jpg&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">HostedGraphite</div><div class="preview-text">Hosted Graphite improves upon standard Graphite. Take a look at how we do this and how Hosted Graphite gives your company better functionality. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Shevaun Frazier</div><div class="thumbnail-date">Nov 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/new-developer-onboarding" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5bfea01e7ffa1b30f88f19c0_Blog-background-organge.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">New Developer Onboarding</div><div class="preview-text">Discussions about onboarding tend to revolve around new hires, for obvious reasons, but the process is important for everyone: while the new developer learns the most important aspects of team and company culture, the team has an opportunity to learn new ideas from a fresh pair of eyes. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Heather Wiencko</div><div class="thumbnail-date">Sep 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/pug-life-how-we-run-a-grafana-instance-for-each-user-with-docker" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5d643d90b1c59597a144484a_pug-header.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">PUG Life: How we run a Grafana instance for each user with Docker </div><div class="preview-text">The story of Hosted Grafana at Hosted Graphite, and how we run multiple instances of Grafana. Start monitoring with Hosted Graphite and use our Grafana dashboards directly in-platform. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciarán Finn</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/incident-postmortem-template" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Our incident postmortem template</div><div class="preview-text">Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Aug 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/its-dead-jim-how-we-write-an-incident-postmortem" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">&quot;It&#x27;s dead, Jim&quot;: How we write an incident postmortem</div><div class="preview-text">How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Jul 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/surviving-on-call-tips-from-a-hosted-graphite-sre" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c48a4405b91aaf448087677_daveblog-header.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Surviving On-Call: Tips from a Hosted Graphite SRE</div><div class="preview-text">Get an insight into what Hosted Graphite&#x27;s SRE Dave Fennell has learned when being On-Call. Read the tips that he has to offer to make the On-Call experience a positive one.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Dave Fennell</div><div class="thumbnail-date">Jan 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/a-victim-of-its-own-popularity-scaling-our-cloudwatch-integration" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">A victim of its own popularity: Scaling our CloudWatch integration</div><div class="preview-text">Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Egan</div><div class="thumbnail-date">Dec 2018</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/deadlines-lies-and-videotape-the-tale-of-a-grpc-bug" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Deadlines lies and videotape: The tale of a gRPC bug</div><div class="preview-text">If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released).</div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Gaffney</div><div class="thumbnail-date">Nov 2020</div></div></a></div></div></div></div></div><div class="main-section dark"><div class="container-13 w-container"><div class="section-heading"><h2 class="white">See why thousands of engineers trust Hosted Graphite with their monitoring</h2><div class="med-divider"></div></div><a href="https://www.hostedgraphite.com/accounts/signup/" class="get-started-ppc-button w-button">START A FREE TRIAL</a><div class="div-block-14 w-hidden-medium w-hidden-small w-hidden-tiny"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png" width="120" sizes="(max-width: 991px) 100vw, 120px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1600.png 1600w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png 2000w" alt="" class="image-5"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png" width="190" sizes="(max-width: 991px) 100vw, 190px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png 1280w" alt="" class="image-3"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1)-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png 620w" alt="" class="image-4"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png 800w" alt="" class="image-6"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd52f6b273e0001a3d86c_Tableau%20(1).svg" alt="" class="image-7"/></div></div><div class="footer-section"><div class="w-container"><div>© 2024 Metricfire Limited. <br/>All Rights Reserved.<br/><br/><br/>5940 S Rainbow Blvd Ste 400<br/>Las Vegas, NV 89118-2507<br/>United States</div></div></div></div><div class="w-embed"></div><div class="w-embed w-script"><script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Article",
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "https://www.hostedgraphite.com/blog/per-metric-rate-limiting-how-we-protect-our-backend"
},
"headline": "Per-metric rate limiting: How we protect our backend",
"image": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad096b41d8156956a76d15b_screenshot_2017-08-09_16-51-29.png",
"thumbnailUrl": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad096b41d8156956a76d15b_screenshot_2017-08-09_16-51-29.png",
"author": {
"@type": "Person",
"name": "Ciarán Finn",
"url": "ciaran-finn"
},
"publisher": {
"@type": "Organization",
"name": "Metricfire Limited",
"logo": {
"@type": "ImageObject",
"url": "https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg"
}
},
"datePublished": "Aug 15, 2017",
"dateModified": "Sep 28, 2021",
"description": "Learn how Hosted Graphite limits the amount of data received to protect the backend. Learn how the per-metric rate-limiting was built from the ground up."
}
</script>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "SoftwareApplication",
"applicationCategory": "BusinessApplication",
"name": "Hosted Graphite",
"description": "Hosted Graphite offers powerful monitoring and alerting for various business applications, providing integrations with multiple data sources, a Grafana UI, and great customer support.",
"review": [
{
"@type": "Review",
"author": { "@type": "Person", "name": "Cecily G." },
"reviewBody": "This system is the tool every engineer needs to be successful. Easy to use and captures metrics that no other system has successfully been able to pull for us.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Christopher C." },
"reviewBody": "Great product suite, outstanding customer service! Running a distributed data pipeline is challenging to keep healthy, and Hosted Graphite has been an instrumental partner for monitoring system health and getting ahead of issues.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "UC" },
"reviewBody": "A great service with excellent support. Simple to use, inexpensive, and great support.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Brendan C." },
"reviewBody": "High-quality metrics hosting. Hosted Graphite enables us to instrument our products and processes efficiently. We can proactively monitor for issues and use dashboards to identify more minor performance problems before they become major ones.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Jacobson M." },
"reviewBody": "Perfect for remote device analytics. Hosted graphite makes it easy for me to keep track of my remote devices being used by my customers. I can't always be on site, so this allows easy tracking of device details.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "James C." },
"reviewBody": "The easiest way for DevOps to incorporate Kubernetes without having to suffer a paradigm shift away from using Graphite. Prometheus, while academically good, forces security worst practices which were unacceptable for us as a SOC2 compliant organization. Graphite is battle-proven and already integrated, so Hosted Graphite allowed us to do what we needed without drastic changes.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
"datePublished": "2021-03-19"
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Anne M." },
"reviewBody": "Offers great hosting service, Hosted Graphite is quite easy to use and highly secure. I like how they offer the trial version to evaluate if the solution fits your needs. In terms of setup, it's easy with minimal registration details and flexible pricing.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
"datePublished": "2019-02-20"
}
],
"operatingSystem": "All major operating systems",
"aggregateRating": {
"@type": "AggregateRating",
"reviewCount": 7,
"ratingValue": 4.85,
"bestRating": 5,
"worstRating": 1
},
"offers": {
"@type": "Offer",
"price": 19.00,
"priceCurrency": "USD",
"availability": "https://schema.org/InStock",
"url": "https://hostedgraphite.com",
"seller": {
"@type": "Organization",
"name": "Hosted Graphite"
}
}
}
</script></div><script src="https://d3e54v103j8qbb.cloudfront.net/js/jquery-3.5.1.min.dc5e7f18c8.js?site=5a57aa76d1fa2300015ca244" type="text/javascript" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.36b8fb49256177c8.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.83de43c5e9eda212.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.54e1a7f0.3d4248ede2db61fe.js" type="text/javascript"></script><!-- Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-NT79GSP"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) --><script>
img = document.querySelectorAll('.w-embed');
img.forEach(element => element.style.textAlign = 'center');
</script></body></html>