354 lines
52 KiB
HTML
354 lines
52 KiB
HTML
<!DOCTYPE html><!-- Last Published: Wed Jul 02 2025 01:43:30 GMT+0000 (Coordinated Universal Time) --><html data-wf-domain="webflow.hostedgraphite.com" data-wf-page="5ce3e821b5d6678199f31df7" data-wf-site="5a57aa76d1fa2300015ca244" data-wf-collection="5ce3e821b5d6674c4bf31e14" data-wf-item-slug="deadlines-lies-and-videotape-the-tale-of-a-grpc-bug"><head><meta charset="utf-8"/><link href="https://cdn.prod.website-files.com" rel="preconnect" crossorigin="anonymous"/><title>Deadlines lies and videotape: The tale of a gRPC bug</title><meta content="If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released)." name="description"/><meta content="Deadlines lies and videotape: The tale of a gRPC bug" property="og:title"/><meta content="If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released)." property="og:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png" property="og:image"/><meta content="Deadlines lies and videotape: The tale of a gRPC bug" name="twitter:title"/><meta content="If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released)." name="twitter:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png" name="twitter:image"/><meta property="og:type" content="website"/><meta content="summary_large_image" name="twitter:card"/><meta content="width=device-width, initial-scale=1" name="viewport"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/css/hosted-graphite-blog.webflow.shared.f88b646c4.min.css" rel="stylesheet" type="text/css"/><link href="https://fonts.googleapis.com" rel="preconnect"/><link href="https://fonts.gstatic.com" rel="preconnect" crossorigin="anonymous"/><script src="https://ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js" type="text/javascript"></script><script type="text/javascript">WebFont.load({ google: { families: ["Open Sans:300,300italic,400,400italic,600,600italic,700,700italic,800,800italic","Inconsolata:400,700","Montserrat:100,100italic,200,200italic,300,300italic,400,400italic,500,500italic,600,600italic,700,700italic,800,800italic,900,900italic","Lato:100,100italic,300,300italic,400,400italic,700,700italic,900,900italic","Droid Sans:400,700","Vollkorn:400,400italic,700,700italic","Ubuntu:300,300italic,400,400italic,500,500italic,700,700italic","Lora:regular,italic,700","Oxygen:300,regular,700","Source Sans Pro:200,200italic,300,300italic,regular,600,600italic,700,700italic,900,900italic"] }});</script><script type="text/javascript">!function(o,c){var n=c.documentElement,t=" w-mod-";n.className+=t+"js",("ontouchstart"in o||o.DocumentTouch&&c instanceof DocumentTouch)&&(n.className+=t+"touch")}(window,document);</script><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f272bbb5a83b7b3239ad6_32.png" rel="shortcut icon" type="image/x-icon"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f27323f9c40d523c9bfcc_256.png" rel="apple-touch-icon"/><script src="https://cdn.jsdelivr.net/npm/code-prettify@0.1.0/src/prettify.min.js"></script>
|
||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
|
||
<script type="text/javascript">
|
||
var embeddify = function(){
|
||
$("h6").each(function(index){
|
||
var h6 = $(this);
|
||
var url = h6.text();
|
||
console.log("Embeddifying " + url);
|
||
h6.hide();
|
||
var embed_message = $('<pre>Embedding <a href="'+url+'" target="_blank">some code</a>, one moment...</pre>');
|
||
embed_message.insertAfter(h6);
|
||
$.ajax(url, {
|
||
success: function(content){
|
||
$('<?prettify linenums=true?><pre class="prettyprint">'+content+'</pre>').insertAfter(h6);
|
||
embed_message.hide();
|
||
PR.prettyPrint();
|
||
},
|
||
error: function(jqXHR, textStatus, errorThrown){
|
||
embed_message.html('Tried to fetch <a href="'+url+'" target="_blank">some code from GitHub</a>, but it failed with an error: "'+errorThrown+'"');
|
||
}
|
||
});
|
||
});
|
||
};
|
||
|
||
var webflow_removed = false;
|
||
var attempts_remaining = 10;
|
||
|
||
function dewebflow() {
|
||
/* Temporarily remove this badge while we're sorting out
|
||
serving this from the right domains so the hosting provider
|
||
will remove the badge properly. */
|
||
|
||
var badges = $("a.w-webflow-badge");
|
||
if(badges.length == 1)
|
||
{
|
||
$("a.w-webflow-badge").remove();
|
||
webflow_removed = true;
|
||
} else {
|
||
attempts_remaining--;
|
||
}
|
||
|
||
if(!webflow_removed || attempts_remaining > 0)
|
||
setTimeout(dewebflow, 500);
|
||
}
|
||
|
||
$(document).ready(embeddify);
|
||
$(document).ready(dewebflow);
|
||
|
||
</script>
|
||
|
||
<style type"text/css">
|
||
/* Turn on line numbering for every line of prettyprinted code, instead of every fifth line. */
|
||
li.L0, li.L1, li.L2, li.L3,
|
||
li.L5, li.L6, li.L7, li.L8 {
|
||
list-style-type: decimal !important;
|
||
}
|
||
</style>
|
||
|
||
<!-- Google Tag Manager -->
|
||
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||
})(window,document,'script','dataLayer','GTM-NT79GSP');</script>
|
||
<!-- End Google Tag Manager --></head><body class="body-9"><div data-collapse="medium" data-animation="default" data-duration="200" data-easing="ease" data-easing2="ease" role="banner" class="navbar-9 w-nav"><div class="section-15"><div class="container-12 w-container"><nav role="navigation" class="nav-menu-5 w-clearfix w-nav-menu"><a href="https://www.hostedgraphite.com/accounts/signup/" class="hg-nav-link right marketing get-started-ppc-button w-nav-link">Get started free</a><a href="https://www.hostedgraphite.com/accounts/login/" class="hg-nav-link right w-nav-link">Login</a><a href="https://www.hostedgraphite.com/enterprise" class="hg-nav-link w-nav-link">Enterprise</a><a href="https://www.hostedgraphite.com/pricing" class="hg-nav-link w-nav-link">Pricing</a><a href="https://www.hostedgraphite.com/docs/" class="hg-nav-link w-nav-link">Docs</a><a href="https://www.hostedgraphite.com/customers" class="hg-nav-link w-hidden-medium w-hidden-small w-hidden-tiny w-nav-link">Customers</a><a href="https://www.hostedgraphite.com/product" class="hg-nav-link w-nav-link">Features</a></nav><div class="menu-button-3 w-nav-button"><div class="w-icon-nav-menu"></div></div></div></div><div class="div-block-15"><a href="https://www.hostedgraphite.com" class="brand-4 w-nav-brand"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg" width="1088.5" alt="" class="image-52"/></a></div><div class="section-16"></div></div><div data-w-expand="category" style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png")" class="hero-blog"></div><div class="main-section"><div class="w-container"><div class="section-heading"><h1 class="blog-post-title">Deadlines lies and videotape: The tale of a gRPC bug</h1><div class="blog-date">November 18, 2020</div><a data-w-expand="category" href="/blog-categories/engineering" class="blog-category">Engineering</a><div class="full-divide"></div><div class="w-embed w-script"><script>
|
||
document.addEventListener("DOMContentLoaded", function() {
|
||
const tocContainer = document.createElement('div');
|
||
tocContainer.id = 'toc-container';
|
||
tocContainer.innerHTML = '<h2>Table of Contents</h2><ul id="toc"></ul>';
|
||
|
||
const sectionHeading = document.querySelector('.section-heading');
|
||
sectionHeading.insertAdjacentElement('afterend', tocContainer);
|
||
|
||
const toc = document.getElementById('toc');
|
||
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
|
||
|
||
headers.forEach((header) => {
|
||
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
|
||
header.id = id;
|
||
|
||
const li = document.createElement('li');
|
||
const a = document.createElement('a');
|
||
a.href = `#${id}`;
|
||
a.textContent = header.textContent;
|
||
li.appendChild(a);
|
||
|
||
toc.appendChild(li);
|
||
});
|
||
|
||
document.querySelectorAll('#toc a').forEach(anchor => {
|
||
anchor.addEventListener('click', function(e) {
|
||
e.preventDefault();
|
||
document.querySelector(this.getAttribute('href')).scrollIntoView({
|
||
behavior: 'smooth'
|
||
});
|
||
history.pushState(null, null, this.getAttribute('href'));
|
||
});
|
||
});
|
||
});
|
||
</script>
|
||
<style>
|
||
#toc-container {
|
||
text-align: start;
|
||
width: 85%;
|
||
margin: 0 auto 3em;
|
||
}
|
||
#toc-container h2 {
|
||
margin-top: 0;
|
||
}
|
||
#toc-container ul {
|
||
padding-left: 1em;
|
||
}
|
||
#toc-container li {
|
||
margin-bottom: 0.5em;
|
||
}
|
||
#toc li a {
|
||
text-decoration: none;
|
||
color: #2e2e2e;
|
||
}
|
||
#toc li a:hover {
|
||
text-decoration: underline;
|
||
}
|
||
</style></div></div><div class="w-condition-invisible w-embed"></div><div class="blog-content"><div class="blog-post w-richtext"><p>By Ciaran Gaffney and Fran Garcia</p><p></p><p>Recently, we’ve been working on a project to decouple some parts of our ingestion and aggregation pipeline to make it more resilient against network failures. As a result, we needed a mechanism to transfer data as it’s processed to these new components. This is a system that’s subjected to high throughput, with reliability being an important factor. Due to these requirements, we ended up settling on <a href="https://grpc.io/">gRPC</a> to drive traffic through this pipeline.</p><p>The component in question was due to be broken down into two pieces, which we’ll call layer0 and layer1. Layer0 would receive data directly from our ingestion pipeline, do a bit of preprocessing, and forward it to layer1 for further processing. Any delays in data reaching layer1 would also result in delays for our users to view their own data (<a href="https://www.hostedgraphite.com/">we run a Hosted Graphite monitoring service</a>). As a result, one of our design priorities was to make sure that a network failure, or a node (or set of nodes) experiencing issues wouldn’t delay processing. Any single layer0 node can have hundreds of layer1 destinations.</p><p>Conveniently, gRPC has the concept of a <a href="https://grpc.io/blog/deadlines">deadline</a>, which allows us to be explicit in how much we’re willing to wait for a single RPC call to be handled by the server. It’s important to note that by default, this deadline is set to unlimited, which is almost certainly not what you want. We therefore highly recommend testing your application and setting a deadline that works for your service.</p><p><a href="https://www.hostedgraphite.com/">HostedGraphite</a> is a service that runs time-series metrics monitoring for application, infrastructure, and systems monitoring. <a href="https://www.hostedgraphite.com/">HostedGraphite</a> can capture any data point and pull it from our awesome <a href="https://www.metricfire.com/dashboards/">Hosted Grafana</a> dashboards. <a href="https://www.hostedgraphite.com/accounts/signup">Try it out with a free trial of Hosted Graphite by signing up here! </a>Alternatively, <a href="https://calendly.com/metricfire-sales/hostedgraphite-demo">get our team on video chat by booking a demo.</a></p><p></p><h2><strong>Do you think someone would do that, just go on the internet and experience timeouts?</strong></h2><p>When we started rolling out the new components, everything was working as expected–with one exception. Shortly after the rollout, we started noticing some strange behaviour. At seemingly random intervals, our layer0 service would stop processing and forwarding data to all layer1 destinations. This would continue for what seemed like random periods, from a few seconds up to minutes. Our <a href="https://www.hostedgraphite.com/">health-checking mechanisms</a> detected this quickly and routed traffic around the affected nodes until they recovered, but this certainly warranted further investigation.</p><p>At this point, we didn’t know what triggered the issue and our instrumentation wasn’t much help. It essentially showed our layer0 service freezing, and even stopped reporting its own internal metrics, only to recover a few minutes later. As we didn’t know how to reproduce the problem, it was hard to make progress.</p><p>Shortly after, we noticed something interesting. During an incident, network connectivity between some layer0 and layer1 nodes had been severely affected. Our system had been designed to handle this eventuality, but the resulting behaviour was not what we expected. If a single layer1 node experiences connectivity issues, we would only expect to see an increased backlog to that particular destination, while others should continue processing traffic as normal. Instead, the whole layer0 would freeze completely. Once connectivity was restored, all layer0 nodes recovered and started working through their backlog as if nothing had ever happened.</p><p></p><div class="w-embed"><picture>
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/Deadlines%2C+lies+and+videotape%3A+The+tale+of+a+gRPC+bug+/panel1.webp" type="image/webp">
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/Deadlines%2C+lies+and+videotape%3A+The+tale+of+a+gRPC+bug+/panel1.jpg" type="image/jpeg">
|
||
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/Deadlines%2C+lies+and+videotape%3A+The+tale+of+a+gRPC+bug+/panel1.webp" alt="A screenshot of a Hosted Graphite dashboard panel monitoring backlog">
|
||
<figcaption>Impact of connectivity issues to our layer0. If our original gRPC deadline was respected, this should not be higher than a few seconds.</figcaption>
|
||
</picture></div><p></p><p>The above image shows that our original gRPC deadline was not being respected, pushing the backlog above 3 minutes.</p><p>We expected that the layer0 service would notice that RPC calls to that particular layer1 destination were not succeeding within our gRPC deadline, and that our internal health-checking mechanisms would disable that destination and route the flow of data around it.</p><h2><strong>The game’s afoot!</strong></h2><p>But at least now we had a clue. When network connectivity was broken between layer0 and layer1 nodes our gRPC deadlines did not seem to be enforced. That also gave us a way to reproduce the issue: we could just use iptables to break connectivity between two otherwise healthy nodes and the problem would appear.</p><p>We then suspected the issue to be within gRPC, but we didn’t have definitive proof, nor any useful information we could give the gRPC team. We needed to dig deeper. First, we started to trace our program with both <a href="https://linux.die.net/man/1/strace">strace</a> and <a href="https://github.com/khamidou/lptrace">lptrace</a>, to see if the blocking would happen within our code or gRPCs. The <a href="https://gist.github.com/gaffer-93/5dada82cecaf56731ad402fee312f94e">lptrace output</a> suggested the blocking happened deep within gRPC. This revealed that the blocking call was whatever function call preceded the call to _check_call_error which, as the output suggests, is a function belonging to gRPC’s channel object.</p><p></p><h2><strong>It’s not you, it’s me</strong></h2><p>Once it emerged that the blocking itself wasn’t happening within our code, we needed a simple repro case before opening an issue with the gRPC project. After all, our own application uses gRPC in complex ways with many different destinations. It was therefore likely that the bug was being triggered by something we were doing on our end. Besides, we asked ourselves, if all it takes to trigger this issue is a break in network connectivity, why aren’t other people reporting it?</p><p>Turns out we just needed some minor <a href="https://gist.github.com/gaffer-93/d8b9d5d392cc20cf3364b42be981f1f9">modifications</a> to the greeter client in their <a href="https://github.com/grpc/grpc/tree/master/examples/python/helloworld">helloworld</a> example to reproduce the issue…almost.</p><p>As it happened, our initial attempts to reproduce the issue with our modified greeter client were unsuccessful. The client would detect that the RPC calls were not succeeding within the stated deadline and fail accordingly, instead of blocking. As a result, we were concerned that the issue had a more subtle trigger that we were missing. We started making some modifications to the client code to make it more similar in behaviour to our service (while still being a minimal repro case) and we found that the size of the message we were trying to send was relevant.</p><p>When we modified our client to send a larger message, we found that blocking happened almost immediately when connectivity was broken. It took much longer to reproduce the issue when sending smaller messages (which made the issue harder to detect during our initial tests).</p><p>At this point, we already had enough information to make a reasonably detailed <a href="https://github.com/grpc/grpc/issues/15889">bug report</a> with the gRPC project. However, we were still curious about this behaviour and wanted to understand the problem better, so we did some more tracing on our modified greeter client. The full results of our investigations are outlined in this <a href="https://github.com/grpc/grpc/issues/15889#issuecomment-401334318">issue comment </a>– we found that the trigger for the blocking was the TCP send buffer filling up.</p><p>Every time the client tries to send a message it gets added to the TCP send buffer, so the <a href="https://linux.die.net/man/2/sendmsg">sendmsg</a> system call technically succeeds. This is normal behaviour: in a healthy system, the kernel would make sure that the messages in the send buffer reach their destination. Seeing as in our case connectivity is broken, the send buffer will continue growing every time we attempt to send a new message (or retry sending a previous message) until the send queue is full. At that point, the sendmsg syscall will fail, and that’s when the blocking happens. It will stay that way until either connectivity is restored, or the kernel gives up on that particular connection, closing it.</p><p>This was an important find for us for two reasons:</p><ul role="list"><li>It provided more information for the gRPC team which would hopefully make it easier to identify and fix the underlying bug.</li><li>It explained some of the behaviours we were experiencing, such as why the service would unblock itself after connectivity was restored, and why any blocking period wouldn’t last more than 20 minutes. This is because the number of retries the kernel is willing to do before giving up on a connection is governed by net.ipv4.tcp_retries2. The timeout between retries is calculated dynamically and is based on the <a href="https://en.wikipedia.org/wiki/Round-trip_delay_time">RTT</a> of the connection but with our defaults results in around 15-20 minutes in total.</li></ul><h2></h2><h2><strong>So you’re telling me there’s a chance…</strong></h2><p>The response from the <a href="https://grpc.io/">gRPC team</a> was great, and soon they had a patch due to be included in their 1.14 release. Unfortunately, the fix had to be reverted as it was causing some tests to fail and was therefore removed from the 1.14 release. It was then suggested that we could rely on <a href="https://github.com/grpc/grpc/blob/master/doc/keepalive.md">keepalive</a> to detect when a connection breaks. When keepalive detects that a channel’s connection is broken it will change the state of the channel, which we can use before sending data through it to decide to back off. This wouldn’t completely solve our problem, but would greatly reduce the likelihood of this issue affecting us.</p><p>To address this issue, the gRPC introduced the following proposal: <a href="https://github.com/grpc/proposal/blob/master/A18-tcp-user-timeout.md">TCP user timeout</a>. This proposal would simply take advantage of the TCP_USER_TIMEOUT socket option when sending data. It controls the maximum amount of time that a socket can have unacknowledged data before the connection is forcibly closed by the kernel, allowing us to recover faster in this kind of scenario. We have been testing the new changes within our services and, thanks to the combination of keepalive and TCP_USER_TIMEOUT, we could no longer reproduce the issue.</p><p></p><h2>“Yes Mister Frodo, it’s over now”</h2><p>Turns out, it wasn’t quite over.</p><p>All our testing around our original repro case indicated the original issue was gone so we rolled out this new version of gRPC along with our services, expecting the backlog buildups to be a thing of the past.</p><p>Not long after, we noticed that when connectivity would break, we were actually experiencing very similar behaviour to before, even with a fully upgraded gRPC. The only difference was that now our gRPC channels weren’t blocking forever (or up to twenty minutes), instead the channel would recover after (almost exactly) 127 seconds. Our first impulse was to verify that TCP_USER_TIMEOUT was correctly being set on our socket options when initiating a channel connection, which it was.</p><p>One thing that’s important to know about TCP_USER_TIMEOUT is that its value is only ever honoured for currently open connections, which means that we were probably timing out before the channel was able to establish a connection. An important kernel setting that can affect how long it takes to timeout when trying to establish a connection is <a href="https://github.com/torvalds/linux/blob/master/Documentation/networking/ip-sysctl.txt#L633">tcp_syn_retries</a> (and its sister socket option, TCP_SYNCNT). This value governs how many times we will try to retransmit a SYN packet before we give up on a connection attempt if our SYN attempts go unacknowledged.</p><p>When attempting to establish a connection, the kernel will retry sending SYN packets and exponentially backing off between retries. The first retry will happen 1 second after the first SYN attempt, with backoff time doubling for each subsequent retry (2 seconds for the second retry, 4 seconds for the third retry, etc). If we have sent tcp_syn_retries<em> </em>unacknowledged SYN retransmission attempts, then we give up on this connection attempt and close it.</p><p>The exponential backoff means that, with a higher number of retries, it will take much longer for a connection attempt to timeout. The current default on modern kernels is 6, which (as you can see on the table below) results in a connection timeout of 127 seconds.</p><p></p><p><strong></strong></p><p><strong>tcp_syn_retries Max connection timeout</strong></p><p><strong></strong>1 3s<br/>2 7s<br/>3 15s<br/>4 31s<br/>5 63s<br/>6 127s</p><p></p><p>We experimented with different values for tcp_syn_retries and we could easily confirm that setting a low enough number of retries for connection attempts greatly reduced the impact of a connection timeout on our services. In our case, a value of 2 or 3 is usually enough to avoid having a noticeable impact, but the right value for you will depend on your workload and services.</p><p></p><div class="w-embed"><picture>
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/Deadlines%2C+lies+and+videotape%3A+The+tale+of+a+gRPC+bug+/panel2.webp" type="image/webp">
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/Deadlines%2C+lies+and+videotape%3A+The+tale+of+a+gRPC+bug+/panel2.jpg" type="image/jpeg">
|
||
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/Deadlines%2C+lies+and+videotape%3A+The+tale+of+a+gRPC+bug+/panel2.webp" alt="A screenshot of a Hosted Graphite Dashboard panel monitoring gRPC Send Time">
|
||
<figcaption>Shows the 1.13 gRPC channels with tcp_syn_retries set to 6, deadlines are clearly not being respected when sending data during network instability.</figcaption>
|
||
</picture></div><div class="w-embed"><picture>
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/Deadlines%2C+lies+and+videotape%3A+The+tale+of+a+gRPC+bug+/panel3.webp" type="image/webp">
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/Deadlines%2C+lies+and+videotape%3A+The+tale+of+a+gRPC+bug+/panel3.jpg" type="image/jpeg">
|
||
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/Deadlines%2C+lies+and+videotape%3A+The+tale+of+a+gRPC+bug+/panel3.webp" alt="A screenshot of a Hosted Graphite dashboard panel monitoring multipe tcp_syn_retries">
|
||
<figcaption>Shows the 1.16 gRPC channels with tcp_syn_retries set to 2, here gRPC is behaving exactly as you would expect, timing out in a predictable fashion. Note the spike to 500ms when the connectivity breaks initially, this is an example of a TCP_USER_TIMEOUT which we currently have set slightly higher than our deadline of 300ms.</figcaption>
|
||
</picture></div><p></p><p>This was great news, and by changing this setting we’re now able to minimise the impact of connectivity issues across our fleet. Unfortunately, this currently requires tweaking a kernel setting that affects your whole system and might not be an appropriate value for all services. The ideal solution would be to have enough control over the socket options gRPC sets so we can set the value of the TCP_SYNCNT socket option and leave the kernel defaults unchanged.</p><p>After some digging into the issues of the gRPC project, we found <a href="https://github.com/grpc/grpc/issues/14685">this bug report</a>. The mention of gRPC “hanging” for 127 seconds when trying to send to unavailable hosts indicates that it’s a special case of our <a href="https://github.com/grpc/grpc/issues/15889">original bug report</a>, that only affects channel connection attempts that are already unavailable. Hopefully, this will result in the TCP_SYNCNT socket option being exposed through gRPC.</p><p></p><h2>So what do I need to do?</h2><p>Once gRPC is upgraded to 1.16.0 or greater, if you are running Python in Linux like us, simply upgrading to gRPC v1.16.0 via pip isn't enough as gRPC's python library, grpcio, relies on a compile-time kernel version check in the GRPC core's C headers which determines whether TCP_USER_TIMEOUT is supported. This <a href="https://github.com/grpc/grpc/blob/62a16fb45a7a01ca0e06b75184c5935b3d35ca2b/src/core/lib/iomgr/port.h#L87">KERNEL_VERSION</a> function is not accessible when building in manylinux1, this leaves <a href="https://github.com/grpc/grpc/blob/62a16fb45a7a01ca0e06b75184c5935b3d35ca2b/src/core/lib/iomgr/port.h#L88">GRPC_HAVE_TCP_USER_TIMEOUT</a> unset and restricts grpcio from passing TCP_USER_TIMEOUT as a socket option. As the conversation on the <a href="https://github.com/grpc/grpc/issues/17206">issue #17206 </a>suggests the only way to get TCP_USER_TIMEOUT support is to build grpcio from the source for your target Linux distribution. Details on how to build grpcio from source can be found in <a href="https://github.com/grpc/grpc/tree/master/src/python/grpcio#from-source">grpcio's documentation</a>.</p><p>If you already have client-side keepalive enabled, there’s no need to do anything special to avail of the new TCP timeout, gRPC will automatically enable and configure it for you. If you don’t have keepalive enabled in your gRPC applications, we highly recommend that you do it. Here’s a <a href="https://gist.github.com/gaffer-93/84d19d9ac1b882df0e411b04c4f9983f">modified version of the greeter client</a> of gRPC’s helloworld example, showing how you can enable keepalive and in turn set the TCP_USER_TIMEOUT socket option.</p><p>There are two channel options that you’ll need to set to enable keepalive on your gRPC client:</p><ul role="list"><li>grpc.keepalive_time_ms: How often keepalive pings should be sent by your client.</li><li>grpc.keepalive_timeout_ms: How much we should wait for a keepalive ping to be acknowledged before considering this connection as unhealthy and closing it. TCP_USER_TIMEOUT will also be set to this value for the underlying socket.</li></ul><p>The actual values for both settings will greatly depend on your own network and application. Shorter values will help you detect issues faster but you might end up flooding your own channel with ping messages, and get a bunch of false positives if latencies get a bit high. Find one that makes sense and works for your architecture.</p><p>If you’re worried about connection retries severely impacting your application, you might want to tweak the value of net.ipv4.tcp_syn_retries to something you’re comfortable with (see table above with what effective timeout values to expect for any value), but we recommend being very careful when modifying this value, and only doing so after testing extensively, as it can negatively affect your system if set too low.</p><p>TCP_USER_TIMEOUT has already been <a href="https://github.com/grpc/grpc/pull/16419">merged</a> and is available as part of the 1.16 gRPC release. Work on the second issue is still <a href="https://www.hostedgraphite.com/blog/deadlines-lies-and-videotape-the-tale-of-a-grpc-bug#">ongoing</a>. We’d like to thank the gRPC team for their work on this project and their help with this particular issue.</p><p></p><h2>Conclusion</h2><p>If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released).</p><p>Dealing with gRPC deadlines and gRPC timeouts is a huge pain, and it can be difficult to figure out where the problem lies. Using a monitoring service like <a href="https://www.hostedgraphite.com/">Hosted Graphite</a> can help you monitor where your issues are happening and why. If any resources are being affected by your gRPC you will find out immediately. <a href="https://www.hostedgraphite.com/accounts/signup">Get onto a 14-day free trial with Hosted Graphite to test this out for yourself here.</a><a href="https://calendly.com/metricfire-sales/hostedgraphite-demo">You can also book a demo with the Hosted Graphite team and we'll help you out directly!</a></p></div><div class="sticky-toc-container"><div class="cta-sidebar"><div class="text-block-31"><strong class="bold-text-15">Try Hosted Graphite now!</strong></div><p class="paragraph-12">Get Hosted Graphite free for 14 days. No credit card required.</p><a href="https://www.hostedgraphite.com/accounts/signup/?from=heroku_blogpost" class="cta-side-button w-button">Get Started</a></div><div class="toc w-embed w-script"><script>
|
||
document.addEventListener("DOMContentLoaded", function() {
|
||
function createSideToc(containerId) {
|
||
const container = document.querySelector(containerId);
|
||
|
||
// Create and append the title <h2>
|
||
const tocTitle = document.createElement('h2');
|
||
tocTitle.textContent = 'Table of Contents';
|
||
container.appendChild(tocTitle);
|
||
|
||
const toc = document.createElement('ul');
|
||
toc.className = 'toc';
|
||
|
||
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
|
||
const ids = new Set(); // To ensure unique IDs
|
||
|
||
headers.forEach((header) => {
|
||
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
|
||
header.id = id;
|
||
if (!ids.has(id)) {
|
||
ids.add(id);
|
||
|
||
const li = document.createElement('li');
|
||
const a = document.createElement('a');
|
||
a.href = `#${id}`;
|
||
a.textContent = header.textContent;
|
||
li.appendChild(a);
|
||
|
||
toc.appendChild(li);
|
||
}
|
||
});
|
||
|
||
container.appendChild(toc);
|
||
|
||
// Smooth scroll for the new TOC
|
||
toc.querySelectorAll('a').forEach(anchor => {
|
||
anchor.addEventListener('click', function(e) {
|
||
e.preventDefault();
|
||
document.querySelector(this.getAttribute('href')).scrollIntoView({
|
||
behavior: 'smooth'
|
||
});
|
||
history.pushState(null, null, this.getAttribute('href'));
|
||
});
|
||
});
|
||
}
|
||
|
||
// Create the sticky TOC in the sidebar
|
||
createSideToc('.sticky-toc-container');
|
||
});
|
||
</script>
|
||
|
||
<style>
|
||
/* Specific Styling for Sticky TOC */
|
||
.sticky-toc-container {
|
||
position: sticky;
|
||
top: 110px;
|
||
width: 340px;
|
||
height: 100%;
|
||
overflow-y: auto;
|
||
padding: 15px;
|
||
border-radius: 8px;
|
||
font-family: Arial, sans-serif;
|
||
font-size: 14px;
|
||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
|
||
text-align: left;
|
||
}
|
||
|
||
.sticky-toc-container h2 {
|
||
margin-top: 0;
|
||
font-size: 18px;
|
||
color: #2e2e2e;
|
||
}
|
||
|
||
.sticky-toc-container ul {
|
||
list-style-type: none;
|
||
padding-left: 0;
|
||
margin: 0;
|
||
}
|
||
|
||
.sticky-toc-container li {
|
||
margin-bottom: 0.6em;
|
||
}
|
||
|
||
.sticky-toc-container a {
|
||
text-decoration: none;
|
||
color: #8e8b8b;
|
||
}
|
||
|
||
.sticky-toc-container a:hover {
|
||
text-decoration: underline;
|
||
}
|
||
|
||
/* Make sure the TOC has a scroll bar when needed */
|
||
.toc {
|
||
max-height: 35vh;
|
||
overflow-y: auto;
|
||
}
|
||
|
||
/* Media Query to Hide sticky-toc-container on smaller screens */
|
||
@media (max-width: 1215px) {
|
||
.sticky-toc-container {
|
||
display: none;
|
||
}
|
||
}
|
||
</style></div></div></div><div class="w-condition-invisible w-embed w-script"><script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js"></script>
|
||
<pre class="prettyprint">
|
||
|
||
</pre></div><div class="div-block-2"><div class="full-divide"></div><div class="author-wrapper"><a href="/blog-authors/ciaran-gaffney" data-w-expand="authors" class="author-name">Ciaran Gaffney</a><div class="smallest-divider"></div><div data-w-expand="authors" class="author-bio w-richtext"><p>SRE at Hosted Graphite.</p></div><div><a href="https://twitter.com/hostedgraphite" data-w-expand="authors" class="social-link w-inline-block"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a57aa7ad1fa2300015ca29a_social-18.svg" width="68" alt="" class="image-2"/></a><a data-w-expand="authors" href="#" class="social-link w-inline-block"></a><a href="#" data-w-expand="authors" class="social-link w-inline-block"></a></div></div></div></div></div><div class="main-section gray"><div class="w-container"><div class="section-heading"><h2 class="heading-6">Related Posts</h2><div class="med-divider"></div></div><div class="w-dyn-list"><div role="list" class="w-clearfix w-dyn-items w-row"><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/hosted-graphite-isnt-graphite" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5dd37e9fe192cd2ea5b8af68_Screen%20Shot%202019-11-19%20at%202.14.02%20PM.jpg")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">HostedGraphite</div><div class="preview-text">Hosted Graphite improves upon standard Graphite. Take a look at how we do this and how Hosted Graphite gives your company better functionality. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Shevaun Frazier</div><div class="thumbnail-date">Nov 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/new-developer-onboarding" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5bfea01e7ffa1b30f88f19c0_Blog-background-organge.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">New Developer Onboarding</div><div class="preview-text">Discussions about onboarding tend to revolve around new hires, for obvious reasons, but the process is important for everyone: while the new developer learns the most important aspects of team and company culture, the team has an opportunity to learn new ideas from a fresh pair of eyes. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Heather Wiencko</div><div class="thumbnail-date">Sep 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/pug-life-how-we-run-a-grafana-instance-for-each-user-with-docker" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5d643d90b1c59597a144484a_pug-header.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">PUG Life: How we run a Grafana instance for each user with Docker </div><div class="preview-text">The story of Hosted Grafana at Hosted Graphite, and how we run multiple instances of Grafana. Start monitoring with Hosted Graphite and use our Grafana dashboards directly in-platform. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciarán Finn</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/incident-postmortem-template" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Our incident postmortem template</div><div class="preview-text">Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Aug 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/its-dead-jim-how-we-write-an-incident-postmortem" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">"It's dead, Jim": How we write an incident postmortem</div><div class="preview-text">How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Jul 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/surviving-on-call-tips-from-a-hosted-graphite-sre" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c48a4405b91aaf448087677_daveblog-header.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Surviving On-Call: Tips from a Hosted Graphite SRE</div><div class="preview-text">Get an insight into what Hosted Graphite's SRE Dave Fennell has learned when being On-Call. Read the tips that he has to offer to make the On-Call experience a positive one.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Dave Fennell</div><div class="thumbnail-date">Jan 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/a-victim-of-its-own-popularity-scaling-our-cloudwatch-integration" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">A victim of its own popularity: Scaling our CloudWatch integration</div><div class="preview-text">Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Egan</div><div class="thumbnail-date">Dec 2018</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/status-page-updates-its-all-about-timing" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Status page updates: It’s all about timing</div><div class="preview-text">Check out part 2 of Hosted Graphite's SRE process on how to handing status page updates. Learn more in detail on the responsibilities of communication, duration of updating the page, and who’s responsible for certain tasks.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Oct 2018</div></div></a></div></div></div></div></div><div class="main-section dark"><div class="container-13 w-container"><div class="section-heading"><h2 class="white">See why thousands of engineers trust Hosted Graphite with their monitoring</h2><div class="med-divider"></div></div><a href="https://www.hostedgraphite.com/accounts/signup/" class="get-started-ppc-button w-button">START A FREE TRIAL</a><div class="div-block-14 w-hidden-medium w-hidden-small w-hidden-tiny"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png" width="120" sizes="(max-width: 991px) 100vw, 120px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1600.png 1600w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png 2000w" alt="" class="image-5"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png" width="190" sizes="(max-width: 991px) 100vw, 190px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png 1280w" alt="" class="image-3"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1)-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png 620w" alt="" class="image-4"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png 800w" alt="" class="image-6"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd52f6b273e0001a3d86c_Tableau%20(1).svg" alt="" class="image-7"/></div></div><div class="footer-section"><div class="w-container"><div>© 2024 Metricfire Limited. <br/>All Rights Reserved.<br/><br/><br/>5940 S Rainbow Blvd Ste 400<br/>Las Vegas, NV 89118-2507<br/>United States</div></div></div></div><div class="w-embed"></div><div class="w-embed w-script"><script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "Article",
|
||
"mainEntityOfPage": {
|
||
"@type": "WebPage",
|
||
"@id": "https://www.hostedgraphite.com/blog/deadlines-lies-and-videotape-the-tale-of-a-grpc-bug"
|
||
},
|
||
"headline": "Deadlines lies and videotape: The tale of a gRPC bug",
|
||
"image": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png",
|
||
"thumbnailUrl": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png",
|
||
"author": {
|
||
"@type": "Person",
|
||
"name": "Ciaran Gaffney",
|
||
"url": "ciaran-gaffney"
|
||
},
|
||
"publisher": {
|
||
"@type": "Organization",
|
||
"name": "Metricfire Limited",
|
||
"logo": {
|
||
"@type": "ImageObject",
|
||
"url": "https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg"
|
||
}
|
||
},
|
||
"datePublished": "Nov 18, 2020",
|
||
"dateModified": "Apr 19, 2024",
|
||
"description": "If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released)."
|
||
}
|
||
</script>
|
||
|
||
<script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "SoftwareApplication",
|
||
"applicationCategory": "BusinessApplication",
|
||
"name": "Hosted Graphite",
|
||
"description": "Hosted Graphite offers powerful monitoring and alerting for various business applications, providing integrations with multiple data sources, a Grafana UI, and great customer support.",
|
||
"review": [
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Cecily G." },
|
||
"reviewBody": "This system is the tool every engineer needs to be successful. Easy to use and captures metrics that no other system has successfully been able to pull for us.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Christopher C." },
|
||
"reviewBody": "Great product suite, outstanding customer service! Running a distributed data pipeline is challenging to keep healthy, and Hosted Graphite has been an instrumental partner for monitoring system health and getting ahead of issues.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "UC" },
|
||
"reviewBody": "A great service with excellent support. Simple to use, inexpensive, and great support.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Brendan C." },
|
||
"reviewBody": "High-quality metrics hosting. Hosted Graphite enables us to instrument our products and processes efficiently. We can proactively monitor for issues and use dashboards to identify more minor performance problems before they become major ones.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Jacobson M." },
|
||
"reviewBody": "Perfect for remote device analytics. Hosted graphite makes it easy for me to keep track of my remote devices being used by my customers. I can't always be on site, so this allows easy tracking of device details.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "James C." },
|
||
"reviewBody": "The easiest way for DevOps to incorporate Kubernetes without having to suffer a paradigm shift away from using Graphite. Prometheus, while academically good, forces security worst practices which were unacceptable for us as a SOC2 compliant organization. Graphite is battle-proven and already integrated, so Hosted Graphite allowed us to do what we needed without drastic changes.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
|
||
"datePublished": "2021-03-19"
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Anne M." },
|
||
"reviewBody": "Offers great hosting service, Hosted Graphite is quite easy to use and highly secure. I like how they offer the trial version to evaluate if the solution fits your needs. In terms of setup, it's easy with minimal registration details and flexible pricing.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
|
||
"datePublished": "2019-02-20"
|
||
}
|
||
],
|
||
"operatingSystem": "All major operating systems",
|
||
"aggregateRating": {
|
||
"@type": "AggregateRating",
|
||
"reviewCount": 7,
|
||
"ratingValue": 4.85,
|
||
"bestRating": 5,
|
||
"worstRating": 1
|
||
},
|
||
"offers": {
|
||
"@type": "Offer",
|
||
"price": 19.00,
|
||
"priceCurrency": "USD",
|
||
"availability": "https://schema.org/InStock",
|
||
"url": "https://hostedgraphite.com",
|
||
"seller": {
|
||
"@type": "Organization",
|
||
"name": "Hosted Graphite"
|
||
}
|
||
}
|
||
}
|
||
</script></div><script src="https://d3e54v103j8qbb.cloudfront.net/js/jquery-3.5.1.min.dc5e7f18c8.js?site=5a57aa76d1fa2300015ca244" type="text/javascript" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.36b8fb49256177c8.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.83de43c5e9eda212.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.54e1a7f0.3d4248ede2db61fe.js" type="text/javascript"></script><!-- Google Tag Manager (noscript) -->
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-NT79GSP"
|
||
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
<!-- End Google Tag Manager (noscript) --><script>
|
||
img = document.querySelectorAll('.w-embed');
|
||
img.forEach(element => element.style.textAlign = 'center');
|
||
</script></body></html> |