Files
nexus/sreweekly/articles/134/04-the-secrets-of-load-balancing-long-lived-tcp-connections.html
2026-09-12 17:23:01 +08:00

370 lines
44 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><!-- Last Published: Wed Jul 02 2025 01:43:30 GMT+0000 (Coordinated Universal Time) --><html data-wf-domain="webflow.hostedgraphite.com" data-wf-page="5ce3e821b5d6678199f31df7" data-wf-site="5a57aa76d1fa2300015ca244" data-wf-collection="5ce3e821b5d6674c4bf31e14" data-wf-item-slug="the-secrets-of-load-balancing-long-lived-tcp-connections"><head><meta charset="utf-8"/><link href="https://cdn.prod.website-files.com" rel="preconnect" crossorigin="anonymous"/><title>The Secrets of Load-balancing Long-Lived TCP Connections</title><meta content="Learn how Hosted Graphite managed to load balance customer traffic with the implementation of HAproxy and Linux Virtual Server as TCP and UDP solutions." name="description"/><meta content="The Secrets of Load-balancing Long-Lived TCP Connections" property="og:title"/><meta content="Learn how Hosted Graphite managed to load balance customer traffic with the implementation of HAproxy and Linux Virtual Server as TCP and UDP solutions." property="og:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be1d49a4f8e0ea469ea126c_gb4.png" property="og:image"/><meta content="The Secrets of Load-balancing Long-Lived TCP Connections" name="twitter:title"/><meta content="Learn how Hosted Graphite managed to load balance customer traffic with the implementation of HAproxy and Linux Virtual Server as TCP and UDP solutions." name="twitter:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be1d49a4f8e0ea469ea126c_gb4.png" name="twitter:image"/><meta property="og:type" content="website"/><meta content="summary_large_image" name="twitter:card"/><meta content="width=device-width, initial-scale=1" name="viewport"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/css/hosted-graphite-blog.webflow.shared.f88b646c4.min.css" rel="stylesheet" type="text/css"/><link href="https://fonts.googleapis.com" rel="preconnect"/><link href="https://fonts.gstatic.com" rel="preconnect" crossorigin="anonymous"/><script src="https://ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js" type="text/javascript"></script><script type="text/javascript">WebFont.load({ google: { families: ["Open Sans:300,300italic,400,400italic,600,600italic,700,700italic,800,800italic","Inconsolata:400,700","Montserrat:100,100italic,200,200italic,300,300italic,400,400italic,500,500italic,600,600italic,700,700italic,800,800italic,900,900italic","Lato:100,100italic,300,300italic,400,400italic,700,700italic,900,900italic","Droid Sans:400,700","Vollkorn:400,400italic,700,700italic","Ubuntu:300,300italic,400,400italic,500,500italic,700,700italic","Lora:regular,italic,700","Oxygen:300,regular,700","Source Sans Pro:200,200italic,300,300italic,regular,600,600italic,700,700italic,900,900italic"] }});</script><script type="text/javascript">!function(o,c){var n=c.documentElement,t=" w-mod-";n.className+=t+"js",("ontouchstart"in o||o.DocumentTouch&&c instanceof DocumentTouch)&&(n.className+=t+"touch")}(window,document);</script><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f272bbb5a83b7b3239ad6_32.png" rel="shortcut icon" type="image/x-icon"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f27323f9c40d523c9bfcc_256.png" rel="apple-touch-icon"/><script src="https://cdn.jsdelivr.net/npm/code-prettify@0.1.0/src/prettify.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script type="text/javascript">
var embeddify = function(){
$("h6").each(function(index){
var h6 = $(this);
var url = h6.text();
console.log("Embeddifying " + url);
h6.hide();
var embed_message = $('<pre>Embedding <a href="'+url+'" target="_blank">some code</a>, one moment...</pre>');
embed_message.insertAfter(h6);
$.ajax(url, {
success: function(content){
$('<?prettify linenums=true?><pre class="prettyprint">'+content+'</pre>').insertAfter(h6);
embed_message.hide();
PR.prettyPrint();
},
error: function(jqXHR, textStatus, errorThrown){
embed_message.html('Tried to fetch <a href="'+url+'" target="_blank">some code from GitHub</a>, but it failed with an error: "'+errorThrown+'"');
}
});
});
};
var webflow_removed = false;
var attempts_remaining = 10;
function dewebflow() {
/* Temporarily remove this badge while we're sorting out
serving this from the right domains so the hosting provider
will remove the badge properly. */
var badges = $("a.w-webflow-badge");
if(badges.length == 1)
{
$("a.w-webflow-badge").remove();
webflow_removed = true;
} else {
attempts_remaining--;
}
if(!webflow_removed || attempts_remaining > 0)
setTimeout(dewebflow, 500);
}
$(document).ready(embeddify);
$(document).ready(dewebflow);
</script>
<style type"text/css">
/* Turn on line numbering for every line of prettyprinted code, instead of every fifth line. */
li.L0, li.L1, li.L2, li.L3,
li.L5, li.L6, li.L7, li.L8 {
list-style-type: decimal !important;
}
</style>
<!-- Google Tag Manager -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer','GTM-NT79GSP');</script>
<!-- End Google Tag Manager --></head><body class="body-9"><div data-collapse="medium" data-animation="default" data-duration="200" data-easing="ease" data-easing2="ease" role="banner" class="navbar-9 w-nav"><div class="section-15"><div class="container-12 w-container"><nav role="navigation" class="nav-menu-5 w-clearfix w-nav-menu"><a href="https://www.hostedgraphite.com/accounts/signup/" class="hg-nav-link right marketing get-started-ppc-button w-nav-link">Get started free</a><a href="https://www.hostedgraphite.com/accounts/login/" class="hg-nav-link right w-nav-link">Login</a><a href="https://www.hostedgraphite.com/enterprise" class="hg-nav-link w-nav-link">Enterprise</a><a href="https://www.hostedgraphite.com/pricing" class="hg-nav-link w-nav-link">Pricing</a><a href="https://www.hostedgraphite.com/docs/" class="hg-nav-link w-nav-link">Docs</a><a href="https://www.hostedgraphite.com/customers" class="hg-nav-link w-hidden-medium w-hidden-small w-hidden-tiny w-nav-link">Customers</a><a href="https://www.hostedgraphite.com/product" class="hg-nav-link w-nav-link">Features</a></nav><div class="menu-button-3 w-nav-button"><div class="w-icon-nav-menu"></div></div></div></div><div class="div-block-15"><a href="https://www.hostedgraphite.com" class="brand-4 w-nav-brand"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg" width="1088.5" alt="" class="image-52"/></a></div><div class="section-16"></div></div><div data-w-expand="category" style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be1d49a4f8e0ea469ea126c_gb4.png&quot;)" class="hero-blog"></div><div class="main-section"><div class="w-container"><div class="section-heading"><h1 class="blog-post-title">The Secrets of Load-balancing Long-Lived TCP Connections</h1><div class="blog-date">November 18, 2020</div><a data-w-expand="category" href="/blog-categories/engineering" class="blog-category">Engineering</a><div class="full-divide"></div><div class="w-embed w-script"><script>
document.addEventListener("DOMContentLoaded", function() {
const tocContainer = document.createElement('div');
tocContainer.id = 'toc-container';
tocContainer.innerHTML = '<h2>Table of Contents</h2><ul id="toc"></ul>';
const sectionHeading = document.querySelector('.section-heading');
sectionHeading.insertAdjacentElement('afterend', tocContainer);
const toc = document.getElementById('toc');
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
headers.forEach((header) => {
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
header.id = id;
const li = document.createElement('li');
const a = document.createElement('a');
a.href = `#${id}`;
a.textContent = header.textContent;
li.appendChild(a);
toc.appendChild(li);
});
document.querySelectorAll('#toc a').forEach(anchor => {
anchor.addEventListener('click', function(e) {
e.preventDefault();
document.querySelector(this.getAttribute('href')).scrollIntoView({
behavior: 'smooth'
});
history.pushState(null, null, this.getAttribute('href'));
});
});
});
</script>
<style>
#toc-container {
text-align: start;
width: 85%;
margin: 0 auto 3em;
}
#toc-container h2 {
margin-top: 0;
}
#toc-container ul {
padding-left: 1em;
}
#toc-container li {
margin-bottom: 0.5em;
}
#toc li a {
text-decoration: none;
color: #2e2e2e;
}
#toc li a:hover {
text-decoration: underline;
}
</style></div></div><div class="w-condition-invisible w-embed"></div><div class="blog-content"><div class="blog-post w-richtext"><p>How do you deal with load-balancing customer traffic at the border of your infrastructure when you don’t own the network? Following a series of experiments, I implemented a service that leverages our <a href="https://www.hostedgraphite.com/">internal Graphite monitoring</a> to dynamically weight HAProxy backend servers based on some load measurement.</p><p><a href="https://www.hostedgraphite.com/">Hosted Graphite</a> is a time-series metrics monitoring tool built to monitor applications, infrastructure, networks and systems. <a href="https://www.hostedgraphite.com/accounts/signup/">Try it out yourself with a 14-day free trial with HostedGraphite here!</a> Start monitoring your systems, and get your data visualized in our awesome <a href="https://www.hostedgraphite.com/docs/dashboards/grafana-dashboards.html">Hosted Grafana</a> dashboards.</p><h3><strong>Scaling problems</strong></h3><p>In the early days, we relied on Route53 round-robin DNS for load-balancing data sent to our <a href="https://docs.hostedgraphite.com/sending-metrics/supported-protocols">TCP and UDP endpoints</a>, but the limitations of this approach became more obvious as we scaled. Although DNS is a simple solution and relatively cost-effective for load-balancing, it has several drawbacks such as clients disobeying DNS record TTLs and having no control over client-side DNS caching. To add to that, for services like ours with non-uniform per connection data rates, there was no actual balancing being applied. As a result, as we added more and more servers, our DNS records continued to grow and inevitably became difficult to manage. We started to have regular issues with customers overloading servers with TCP traffic sent over a small number of connections. In such cases, rotating servers out of service to mitigate the impact was a manual process for our SREs and would not solve the issue quickly enough. We needed a better way to load-balance those TCP connections that were long-lived.</p><p>Of course, another major problem with scaling is the cost of monitoring as you scale. If you&#x27;re having trouble scaling your monitoring reach out to us here at Hosted Graphite. <a href="https://www.hostedgraphite.com/accounts/signup-metricfire">Sign up for the free trial here!</a></p><h3><strong>Enter HAProxy</strong></h3><p>We decided to run HAProxy instances on each of our ingestion nodes to perform intra-cluster TCP load balancing. With this setup, we had a traffic path like so:</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/haproxy1.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/haproxy1.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/haproxy1.webp" alt="A diagram showing Hosted Graphite's Intra-cluster HAProxy Load Balancing">
<figcaption>Intra-cluster haproxy load balancing.</figcaption>
</picture></div><p>‍</p><p>The above image shows how the Intra-cluster HAProxy load balancing works. HAProxy load balancing enables the ingestion layer to receive messages from TCP and UDP and transmit them through TCP.</p><p>We began with experiments in round-robin and leastconn balancing methods, the latter balances new connections to a node with the current least active connections and offers the best results. At first, we were pretty happy with how this worked, but knew it would be impossible to truly balance the load across the cluster in this way, as not all connections are created equal (5 data-points/s accepted for one connection…5,000 data-points/s for another). So, for a short period, we rolled with this limited load balancing and continued to rely on DNS for balancing UDP.</p><h3><strong>Building an infrastructure border</strong></h3><p>At this point, we realised we desperately needed to find a way to effectively balance our UDP traffic across our ingestion layer, which was at the border of our infrastructure. Client-side DNS caching was hurting us, we had some customers sending huge amounts of UDP data to a subset of our ingestion layer. This was causing overload issues resulting in backlogged data, which the SRE team were frequently getting paged to investigate. Further digging into our options threw little light on the problem, until a fellow SRE, who was leading the project, decided to build a new load-balancing layer which would act as the infrastructure border. With a border layer of load balancers in place, we could clean out our ever-growing DNS records. It also allowed us to continue to load balance TCP connections with HAProxy, while UDP balancing would be handled by Linux Virtual Server (LVS). The LVS project is a high-performance software load balancer that has existed in the Linux kernel since 2001. Why we chose it for a UDP load-balancing solution is worthy of an entire blog post of its own!</p><h3><strong>Connecting the dots</strong></h3><p>We now had a border layer of load balancers that looked something like this:</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/haproxy2.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/haproxy2.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/haproxy2.webp" alt="A diagram showing Hosted Graphite's LVS and HAProxy Load Balancing Layer">
<figcaption>Border layer of load balancers.</figcaption>
</picture></div><p>‍</p><p>This new load balancing layer opened up another opportunity: to revisit our flawed balancing of TCP connections. As the engineer tasked with the job of achieving TCP traffic convergence, I started by looking at how we could use the tools we already had to balance highly variable traffic rates over long-lived TCP connections. Usefully, HAProxy provides a method for dynamically weighting backend servers via the UDS (Unix Domain Socket) API which allows for new connections to be balanced to nodes based on their weighting, but what should determine the weights? The obvious solution was to use our extensive service monitoring, which we had easy access to through our own Graphite API.</p><p>I implemented a new Python service to be run on each load balancer. This would handle the dynamic weighting of individual servers for a specific HAProxy backend based on the results of some measurement of load. In our case, the “measurement of load” was the overall TCP datapoint rate observed at each node in the ingestion layer – however,  this could be anything! loadavg?, network bandwidth?, disk space? (sure.. why not!).</p><p>Leveraging our Graphite render API, we could weight the forwarding of new connections to each node based on the results of a render query something like:</p><p>‍<em>integral(tcp-service.node-x.datapoints.received)</em></p><p>(cool!).</p><h3><strong>How it works</strong></h3><p>So, how does this all work? The operation loop of a single weighter process is as follows:</p><ul role="list"><li>On initialisation, using the included UDS wrapper, HAProxy_weighter fetches the weight currently set for each server present in the configured HAProxy backend.</li><li>For Every rebalance interval, the weighter renders the configured Graphite target to acquire some measurement of the load currently observed on each relevant server.</li><li>Weights are then calculated for each server by feeding the render results to one of two configurable weighting functions.</li><li>The calculated weights are then set in HAProxy, again using the UDS API.</li></ul><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel3.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel3.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel3.webp" alt="A screenshot of a Hosted Graphite dashboard monitoring TCP traffic accepted at ingestion layer.">
</picture></div><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel4.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel4.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel4.webp" alt="A screenshot of a Hosted Graphite panel monitoring connections balanced by haproxy.">
</picture></div><p>‍</p><p>Each weighting function takes a single data point for each server, aggregated from the render results using either the leading edge or the average of the whole dataset for the series, and returns a weight between 1 and 256. We currently have two options: normalised exponential distribution, and limited proportions which are in use for balancing our TCP connections to our ingestion layer. The weight is determined by calculating the inverse proportions for each value of a list of data points while limiting the minimum datapoint value to <code>1/LIMITING_FACTOR</code> of the average, this avoids lower range outliers being assigned a maximum weight proportion. An <code>AGGRESSIVENESS_FACTOR</code> constant can be raised or lowered to modify the standard deviation between weightings.</p><p>If you&#x27;re interested in what we used to visualize the data, that&#x27;s the <a href="https://www.hostedgraphite.com/docs/dashboards/grafana-dashboards.html">Hosted Grafana</a> dashboards that are a part of our <a href="https://www.hostedgraphite.com/">HostedGraphite</a> offering. <a href="https://www.hostedgraphite.com/accounts/signup">Sign on to our free trial and check it out!‍</a></p><p>‍</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel5.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel5.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel5.webp" alt="A screenshot of a Hosted Graphite dashboard panel monitoring weights set at all loadbalancers for server-0028.">
</picture></div><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel6.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel6.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel6.webp" alt="A screenshot of a Hosted Graphite dashboard panel monitoring weights set by haproxy_weighter at a loadbalancer for each ingestion layer server">
</picture></div><p>‍</p><p>Once the haproxy_weighter was eventually deployed, we could see a significant improvement. Our DNS records are smaller and much easier to manage. We no longer have outliers within our ingestion layer. Where before, we would have seen servers receiving 5 times the average traffic due to a couple of particularly traffic-heavy connections, we now have nicely even per-server traffic rates. TCP traffic convergence was achieved. This system also gives us the ability to automatically react quickly to traffic spikes, routing new connections away from overloaded servers, and saving our SREs from getting paged in the depths of the night.</p><p>‍</p><div class="w-embed"><picture>
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel7.webp" type="image/webp">
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel7.jpg" type="image/jpeg">
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/the-secrets-of-load-balancing-long-lived-tcp-connections/hgpanel7.webp" alt="A screenshot of a Hosted Graphite dashboard panel monitoring TCP traffic convergence">
<figcaption>TCP traffic convergence achieved.</figcaption>
</picture></div><p>‍</p><p>Check out these awesome <a href="https://www.hostedgraphite.com/docs/dashboards/grafana-dashboards.html">Grafana graphs</a> we&#x27;re showing! If you want to have your systems visualized with graphs like this, sign on to the <a href="https://www.hostedgraphite.com/">Hosted Graphite app</a>! <a href="https://www.hostedgraphite.com/accounts/signup">You can get a 14-day free trial here!</a> Also, <a href="https://www.hostedgraphite.com/contact">contact us directly</a> and get us on video chat to talk about your monitoring system!</p><p>‍</p><h2>Conclusion</h2><p>HAProxy load balancing is a critical part of your infrastructure. Scaling your proxies is a big challenge, a challenge that&#x27;s even more difficult when you&#x27;re flying blind. Just like we did in this article, you can use <a href="https://www.hostedgraphite.com/">HostedGraphite</a> to monitor your load balancers. <a href="https://www.hostedgraphite.com/accounts/signup">Sign up for the Hosted Graphite free trial here</a>, and start monitoring your data on our awesome <a href="https://www.hostedgraphite.com/docs/dashboards/grafana-dashboards.html">Hosted Grafana</a> dashboards.</p><p>‍</p></div><div class="sticky-toc-container"><div class="cta-sidebar"><div class="text-block-31"><strong class="bold-text-15">Try Hosted Graphite now!</strong></div><p class="paragraph-12">Get Hosted Graphite free for 14 days. No credit card required.</p><a href="https://www.hostedgraphite.com/accounts/signup/?from=heroku_blogpost" class="cta-side-button w-button">Get Started</a></div><div class="toc w-embed w-script"><script>
document.addEventListener("DOMContentLoaded", function() {
function createSideToc(containerId) {
const container = document.querySelector(containerId);
// Create and append the title <h2>
const tocTitle = document.createElement('h2');
tocTitle.textContent = 'Table of Contents';
container.appendChild(tocTitle);
const toc = document.createElement('ul');
toc.className = 'toc';
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
const ids = new Set(); // To ensure unique IDs
headers.forEach((header) => {
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
header.id = id;
if (!ids.has(id)) {
ids.add(id);
const li = document.createElement('li');
const a = document.createElement('a');
a.href = `#${id}`;
a.textContent = header.textContent;
li.appendChild(a);
toc.appendChild(li);
}
});
container.appendChild(toc);
// Smooth scroll for the new TOC
toc.querySelectorAll('a').forEach(anchor => {
anchor.addEventListener('click', function(e) {
e.preventDefault();
document.querySelector(this.getAttribute('href')).scrollIntoView({
behavior: 'smooth'
});
history.pushState(null, null, this.getAttribute('href'));
});
});
}
// Create the sticky TOC in the sidebar
createSideToc('.sticky-toc-container');
});
</script>
<style>
/* Specific Styling for Sticky TOC */
.sticky-toc-container {
position: sticky;
top: 110px;
width: 340px;
height: 100%;
overflow-y: auto;
padding: 15px;
border-radius: 8px;
font-family: Arial, sans-serif;
font-size: 14px;
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
text-align: left;
}
.sticky-toc-container h2 {
margin-top: 0;
font-size: 18px;
color: #2e2e2e;
}
.sticky-toc-container ul {
list-style-type: none;
padding-left: 0;
margin: 0;
}
.sticky-toc-container li {
margin-bottom: 0.6em;
}
.sticky-toc-container a {
text-decoration: none;
color: #8e8b8b;
}
.sticky-toc-container a:hover {
text-decoration: underline;
}
/* Make sure the TOC has a scroll bar when needed */
.toc {
max-height: 35vh;
overflow-y: auto;
}
/* Media Query to Hide sticky-toc-container on smaller screens */
@media (max-width: 1215px) {
.sticky-toc-container {
display: none;
}
}
</style></div></div></div><div class="w-condition-invisible w-embed w-script"><script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js"></script>
<pre class="prettyprint">
</pre></div><div class="div-block-2"><div class="full-divide"></div><div class="author-wrapper"><a href="/blog-authors/ciaran-gaffney" data-w-expand="authors" class="author-name">Ciaran Gaffney</a><div class="smallest-divider"></div><div data-w-expand="authors" class="author-bio w-richtext"><p>SRE at Hosted Graphite.</p></div><div><a href="https://twitter.com/hostedgraphite" data-w-expand="authors" class="social-link w-inline-block"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a57aa7ad1fa2300015ca29a_social-18.svg" width="68" alt="" class="image-2"/></a><a data-w-expand="authors" href="#" class="social-link w-inline-block"></a><a href="#" data-w-expand="authors" class="social-link w-inline-block"></a></div></div></div></div></div><div class="main-section gray"><div class="w-container"><div class="section-heading"><h2 class="heading-6">Related Posts</h2><div class="med-divider"></div></div><div class="w-dyn-list"><div role="list" class="w-clearfix w-dyn-items w-row"><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/hosted-graphite-isnt-graphite" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5dd37e9fe192cd2ea5b8af68_Screen%20Shot%202019-11-19%20at%202.14.02%20PM.jpg&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">HostedGraphite</div><div class="preview-text">Hosted Graphite improves upon standard Graphite. Take a look at how we do this and how Hosted Graphite gives your company better functionality. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Shevaun Frazier</div><div class="thumbnail-date">Nov 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/new-developer-onboarding" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5bfea01e7ffa1b30f88f19c0_Blog-background-organge.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">New Developer Onboarding</div><div class="preview-text">Discussions about onboarding tend to revolve around new hires, for obvious reasons, but the process is important for everyone: while the new developer learns the most important aspects of team and company culture, the team has an opportunity to learn new ideas from a fresh pair of eyes. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Heather Wiencko</div><div class="thumbnail-date">Sep 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/pug-life-how-we-run-a-grafana-instance-for-each-user-with-docker" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5d643d90b1c59597a144484a_pug-header.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">PUG Life: How we run a Grafana instance for each user with Docker </div><div class="preview-text">The story of Hosted Grafana at Hosted Graphite, and how we run multiple instances of Grafana. Start monitoring with Hosted Graphite and use our Grafana dashboards directly in-platform. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciarán Finn</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/incident-postmortem-template" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Our incident postmortem template</div><div class="preview-text">Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Aug 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/its-dead-jim-how-we-write-an-incident-postmortem" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">&quot;It&#x27;s dead, Jim&quot;: How we write an incident postmortem</div><div class="preview-text">How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Jul 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/surviving-on-call-tips-from-a-hosted-graphite-sre" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c48a4405b91aaf448087677_daveblog-header.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Surviving On-Call: Tips from a Hosted Graphite SRE</div><div class="preview-text">Get an insight into what Hosted Graphite&#x27;s SRE Dave Fennell has learned when being On-Call. Read the tips that he has to offer to make the On-Call experience a positive one.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Dave Fennell</div><div class="thumbnail-date">Jan 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/a-victim-of-its-own-popularity-scaling-our-cloudwatch-integration" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">A victim of its own popularity: Scaling our CloudWatch integration</div><div class="preview-text">Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Egan</div><div class="thumbnail-date">Dec 2018</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/deadlines-lies-and-videotape-the-tale-of-a-grpc-bug" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url(&quot;https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png&quot;)" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Deadlines lies and videotape: The tale of a gRPC bug</div><div class="preview-text">If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released).</div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Gaffney</div><div class="thumbnail-date">Nov 2020</div></div></a></div></div></div></div></div><div class="main-section dark"><div class="container-13 w-container"><div class="section-heading"><h2 class="white">See why thousands of engineers trust Hosted Graphite with their monitoring</h2><div class="med-divider"></div></div><a href="https://www.hostedgraphite.com/accounts/signup/" class="get-started-ppc-button w-button">START A FREE TRIAL</a><div class="div-block-14 w-hidden-medium w-hidden-small w-hidden-tiny"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png" width="120" sizes="(max-width: 991px) 100vw, 120px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1600.png 1600w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png 2000w" alt="" class="image-5"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png" width="190" sizes="(max-width: 991px) 100vw, 190px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png 1280w" alt="" class="image-3"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1)-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png 620w" alt="" class="image-4"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png 800w" alt="" class="image-6"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd52f6b273e0001a3d86c_Tableau%20(1).svg" alt="" class="image-7"/></div></div><div class="footer-section"><div class="w-container"><div>© 2024 Metricfire Limited. <br/>All Rights Reserved.<br/><br/><br/>5940 S Rainbow Blvd Ste 400<br/>Las Vegas, NV 89118-2507<br/>United States</div></div></div></div><div class="w-embed"></div><div class="w-embed w-script"><script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Article",
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "https://www.hostedgraphite.com/blog/the-secrets-of-load-balancing-long-lived-tcp-connections"
},
"headline": "The Secrets of Load-balancing Long-Lived TCP Connections",
"image": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be1d49a4f8e0ea469ea126c_gb4.png",
"thumbnailUrl": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be1d49a4f8e0ea469ea126c_gb4.png",
"author": {
"@type": "Person",
"name": "Ciaran Gaffney",
"url": "ciaran-gaffney"
},
"publisher": {
"@type": "Organization",
"name": "Metricfire Limited",
"logo": {
"@type": "ImageObject",
"url": "https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg"
}
},
"datePublished": "Nov 18, 2020",
"dateModified": "Apr 19, 2024",
"description": "Learn how Hosted Graphite managed to load balance customer traffic with the implementation of HAproxy and Linux Virtual Server as TCP and UDP solutions."
}
</script>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "SoftwareApplication",
"applicationCategory": "BusinessApplication",
"name": "Hosted Graphite",
"description": "Hosted Graphite offers powerful monitoring and alerting for various business applications, providing integrations with multiple data sources, a Grafana UI, and great customer support.",
"review": [
{
"@type": "Review",
"author": { "@type": "Person", "name": "Cecily G." },
"reviewBody": "This system is the tool every engineer needs to be successful. Easy to use and captures metrics that no other system has successfully been able to pull for us.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Christopher C." },
"reviewBody": "Great product suite, outstanding customer service! Running a distributed data pipeline is challenging to keep healthy, and Hosted Graphite has been an instrumental partner for monitoring system health and getting ahead of issues.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "UC" },
"reviewBody": "A great service with excellent support. Simple to use, inexpensive, and great support.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Brendan C." },
"reviewBody": "High-quality metrics hosting. Hosted Graphite enables us to instrument our products and processes efficiently. We can proactively monitor for issues and use dashboards to identify more minor performance problems before they become major ones.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Jacobson M." },
"reviewBody": "Perfect for remote device analytics. Hosted graphite makes it easy for me to keep track of my remote devices being used by my customers. I can't always be on site, so this allows easy tracking of device details.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "James C." },
"reviewBody": "The easiest way for DevOps to incorporate Kubernetes without having to suffer a paradigm shift away from using Graphite. Prometheus, while academically good, forces security worst practices which were unacceptable for us as a SOC2 compliant organization. Graphite is battle-proven and already integrated, so Hosted Graphite allowed us to do what we needed without drastic changes.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
"datePublished": "2021-03-19"
},
{
"@type": "Review",
"author": { "@type": "Person", "name": "Anne M." },
"reviewBody": "Offers great hosting service, Hosted Graphite is quite easy to use and highly secure. I like how they offer the trial version to evaluate if the solution fits your needs. In terms of setup, it's easy with minimal registration details and flexible pricing.",
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
"datePublished": "2019-02-20"
}
],
"operatingSystem": "All major operating systems",
"aggregateRating": {
"@type": "AggregateRating",
"reviewCount": 7,
"ratingValue": 4.85,
"bestRating": 5,
"worstRating": 1
},
"offers": {
"@type": "Offer",
"price": 19.00,
"priceCurrency": "USD",
"availability": "https://schema.org/InStock",
"url": "https://hostedgraphite.com",
"seller": {
"@type": "Organization",
"name": "Hosted Graphite"
}
}
}
</script></div><script src="https://d3e54v103j8qbb.cloudfront.net/js/jquery-3.5.1.min.dc5e7f18c8.js?site=5a57aa76d1fa2300015ca244" type="text/javascript" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.36b8fb49256177c8.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.83de43c5e9eda212.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.54e1a7f0.3d4248ede2db61fe.js" type="text/javascript"></script><!-- Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-NT79GSP"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) --><script>
img = document.querySelectorAll('.w-embed');
img.forEach(element => element.style.textAlign = 'center');
</script></body></html>