354 lines
37 KiB
HTML
354 lines
37 KiB
HTML
<!DOCTYPE html><!-- Last Published: Wed Jul 02 2025 01:43:30 GMT+0000 (Coordinated Universal Time) --><html data-wf-domain="webflow.hostedgraphite.com" data-wf-page="5ce3e821b5d6678199f31df7" data-wf-site="5a57aa76d1fa2300015ca244" data-wf-collection="5ce3e821b5d6674c4bf31e14" data-wf-item-slug="continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second"><head><meta charset="utf-8"/><link href="https://cdn.prod.website-files.com" rel="preconnect" crossorigin="anonymous"/><title>Continuous self-testing at Hosted Graphite — why we send external canaries, every second</title><meta content="At Hosted Graphite, it is of utmost importance to monitor our internal systems. Learn more on how our engineers use different types of canaries for continuous self-testing." name="description"/><meta content="Continuous self-testing at Hosted Graphite — why we send external canaries, every second" property="og:title"/><meta content="At Hosted Graphite, it is of utmost importance to monitor our internal systems. Learn more on how our engineers use different types of canaries for continuous self-testing." property="og:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad08c3e0e97b8494f0194b4_img_20170705_150424_1024-3.jpg" property="og:image"/><meta content="Continuous self-testing at Hosted Graphite — why we send external canaries, every second" name="twitter:title"/><meta content="At Hosted Graphite, it is of utmost importance to monitor our internal systems. Learn more on how our engineers use different types of canaries for continuous self-testing." name="twitter:description"/><meta content="https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad08c3e0e97b8494f0194b4_img_20170705_150424_1024-3.jpg" name="twitter:image"/><meta property="og:type" content="website"/><meta content="summary_large_image" name="twitter:card"/><meta content="width=device-width, initial-scale=1" name="viewport"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/css/hosted-graphite-blog.webflow.shared.f88b646c4.min.css" rel="stylesheet" type="text/css"/><link href="https://fonts.googleapis.com" rel="preconnect"/><link href="https://fonts.gstatic.com" rel="preconnect" crossorigin="anonymous"/><script src="https://ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js" type="text/javascript"></script><script type="text/javascript">WebFont.load({ google: { families: ["Open Sans:300,300italic,400,400italic,600,600italic,700,700italic,800,800italic","Inconsolata:400,700","Montserrat:100,100italic,200,200italic,300,300italic,400,400italic,500,500italic,600,600italic,700,700italic,800,800italic,900,900italic","Lato:100,100italic,300,300italic,400,400italic,700,700italic,900,900italic","Droid Sans:400,700","Vollkorn:400,400italic,700,700italic","Ubuntu:300,300italic,400,400italic,500,500italic,700,700italic","Lora:regular,italic,700","Oxygen:300,regular,700","Source Sans Pro:200,200italic,300,300italic,regular,600,600italic,700,700italic,900,900italic"] }});</script><script type="text/javascript">!function(o,c){var n=c.documentElement,t=" w-mod-";n.className+=t+"js",("ontouchstart"in o||o.DocumentTouch&&c instanceof DocumentTouch)&&(n.className+=t+"touch")}(window,document);</script><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f272bbb5a83b7b3239ad6_32.png" rel="shortcut icon" type="image/x-icon"/><link href="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5b5f27323f9c40d523c9bfcc_256.png" rel="apple-touch-icon"/><script src="https://cdn.jsdelivr.net/npm/code-prettify@0.1.0/src/prettify.min.js"></script>
|
||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
|
||
<script type="text/javascript">
|
||
var embeddify = function(){
|
||
$("h6").each(function(index){
|
||
var h6 = $(this);
|
||
var url = h6.text();
|
||
console.log("Embeddifying " + url);
|
||
h6.hide();
|
||
var embed_message = $('<pre>Embedding <a href="'+url+'" target="_blank">some code</a>, one moment...</pre>');
|
||
embed_message.insertAfter(h6);
|
||
$.ajax(url, {
|
||
success: function(content){
|
||
$('<?prettify linenums=true?><pre class="prettyprint">'+content+'</pre>').insertAfter(h6);
|
||
embed_message.hide();
|
||
PR.prettyPrint();
|
||
},
|
||
error: function(jqXHR, textStatus, errorThrown){
|
||
embed_message.html('Tried to fetch <a href="'+url+'" target="_blank">some code from GitHub</a>, but it failed with an error: "'+errorThrown+'"');
|
||
}
|
||
});
|
||
});
|
||
};
|
||
|
||
var webflow_removed = false;
|
||
var attempts_remaining = 10;
|
||
|
||
function dewebflow() {
|
||
/* Temporarily remove this badge while we're sorting out
|
||
serving this from the right domains so the hosting provider
|
||
will remove the badge properly. */
|
||
|
||
var badges = $("a.w-webflow-badge");
|
||
if(badges.length == 1)
|
||
{
|
||
$("a.w-webflow-badge").remove();
|
||
webflow_removed = true;
|
||
} else {
|
||
attempts_remaining--;
|
||
}
|
||
|
||
if(!webflow_removed || attempts_remaining > 0)
|
||
setTimeout(dewebflow, 500);
|
||
}
|
||
|
||
$(document).ready(embeddify);
|
||
$(document).ready(dewebflow);
|
||
|
||
</script>
|
||
|
||
<style type"text/css">
|
||
/* Turn on line numbering for every line of prettyprinted code, instead of every fifth line. */
|
||
li.L0, li.L1, li.L2, li.L3,
|
||
li.L5, li.L6, li.L7, li.L8 {
|
||
list-style-type: decimal !important;
|
||
}
|
||
</style>
|
||
|
||
<!-- Google Tag Manager -->
|
||
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||
})(window,document,'script','dataLayer','GTM-NT79GSP');</script>
|
||
<!-- End Google Tag Manager --></head><body class="body-9"><div data-collapse="medium" data-animation="default" data-duration="200" data-easing="ease" data-easing2="ease" role="banner" class="navbar-9 w-nav"><div class="section-15"><div class="container-12 w-container"><nav role="navigation" class="nav-menu-5 w-clearfix w-nav-menu"><a href="https://www.hostedgraphite.com/accounts/signup/" class="hg-nav-link right marketing get-started-ppc-button w-nav-link">Get started free</a><a href="https://www.hostedgraphite.com/accounts/login/" class="hg-nav-link right w-nav-link">Login</a><a href="https://www.hostedgraphite.com/enterprise" class="hg-nav-link w-nav-link">Enterprise</a><a href="https://www.hostedgraphite.com/pricing" class="hg-nav-link w-nav-link">Pricing</a><a href="https://www.hostedgraphite.com/docs/" class="hg-nav-link w-nav-link">Docs</a><a href="https://www.hostedgraphite.com/customers" class="hg-nav-link w-hidden-medium w-hidden-small w-hidden-tiny w-nav-link">Customers</a><a href="https://www.hostedgraphite.com/product" class="hg-nav-link w-nav-link">Features</a></nav><div class="menu-button-3 w-nav-button"><div class="w-icon-nav-menu"></div></div></div></div><div class="div-block-15"><a href="https://www.hostedgraphite.com" class="brand-4 w-nav-brand"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg" width="1088.5" alt="" class="image-52"/></a></div><div class="section-16"></div></div><div data-w-expand="category" style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad08c3e0e97b8494f0194b4_img_20170705_150424_1024-3.jpg")" class="hero-blog"></div><div class="main-section"><div class="w-container"><div class="section-heading"><h1 class="blog-post-title">Continuous self-testing at Hosted Graphite — why we send external canaries, every second</h1><div class="blog-date">July 6, 2017</div><a data-w-expand="category" href="/blog-categories/engineering" class="blog-category">Engineering</a><div class="full-divide"></div><div class="w-embed w-script"><script>
|
||
document.addEventListener("DOMContentLoaded", function() {
|
||
const tocContainer = document.createElement('div');
|
||
tocContainer.id = 'toc-container';
|
||
tocContainer.innerHTML = '<h2>Table of Contents</h2><ul id="toc"></ul>';
|
||
|
||
const sectionHeading = document.querySelector('.section-heading');
|
||
sectionHeading.insertAdjacentElement('afterend', tocContainer);
|
||
|
||
const toc = document.getElementById('toc');
|
||
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
|
||
|
||
headers.forEach((header) => {
|
||
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
|
||
header.id = id;
|
||
|
||
const li = document.createElement('li');
|
||
const a = document.createElement('a');
|
||
a.href = `#${id}`;
|
||
a.textContent = header.textContent;
|
||
li.appendChild(a);
|
||
|
||
toc.appendChild(li);
|
||
});
|
||
|
||
document.querySelectorAll('#toc a').forEach(anchor => {
|
||
anchor.addEventListener('click', function(e) {
|
||
e.preventDefault();
|
||
document.querySelector(this.getAttribute('href')).scrollIntoView({
|
||
behavior: 'smooth'
|
||
});
|
||
history.pushState(null, null, this.getAttribute('href'));
|
||
});
|
||
});
|
||
});
|
||
</script>
|
||
<style>
|
||
#toc-container {
|
||
text-align: start;
|
||
width: 85%;
|
||
margin: 0 auto 3em;
|
||
}
|
||
#toc-container h2 {
|
||
margin-top: 0;
|
||
}
|
||
#toc-container ul {
|
||
padding-left: 1em;
|
||
}
|
||
#toc-container li {
|
||
margin-bottom: 0.5em;
|
||
}
|
||
#toc li a {
|
||
text-decoration: none;
|
||
color: #2e2e2e;
|
||
}
|
||
#toc li a:hover {
|
||
text-decoration: underline;
|
||
}
|
||
</style></div></div><div class="w-condition-invisible w-embed"></div><div class="blog-content"><div class="blog-post w-richtext"><p>At Hosted Graphite, internal system monitoring is something our engineers are always concerned about. To detect any degradation we continuously self-test all our endpoints — every second, every day. It gives us an early warning sign of small changes that could be an indicator of larger problems and lets us know how well our service is running. We try to measure what our customers are actually experiencing and this is one of the main metrics of service quality that we grade ourselves on.</p><h3><strong>How does it work?</strong></h3><ol role="list"><li>Each canary service sends a value of '1' once per second with the appropriate timestamp to each endpoint that we accept metric traffic on.</li><li>In the case of any TCP connections we re-establish a new connection each time (and record the time taken) to allow measurement of network delays and services being overloaded.</li><li>We can then continuously monitor these metrics for any drop in value.</li></ol><p>This system lets us know if something’s wrong: when the amount of traffic we record drops below an acceptable loss amount for a particular service, our alerting tools raise the alarm. We can then check on the exact number of failed data-points by looking at our <a href="https://www.hostedgraphite.com/docs/advanced/data-views.html">sum dataviews</a> presented on a dashboard. These figures are aggregated and grouped by protocol and source network, allowing us to determine the extent of the degradation at a glance. We can answer the "Just one AWS zone?" or "All UDP services?" questions extremely quickly because of this continuous, distributed monitoring.</p><p></p><div class="w-embed"><picture>
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second/canaries1.webp" type="image/webp">
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second/canaries1.jpg" type="image/jpeg">
|
||
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second/canaries1.webp" alt="A snippet of a Hosted Graphite canaries dashboard reporting 100% on all protocols">
|
||
<figcaption>A snippet of the canaries dashboard -- all green right now!</figcaption>
|
||
</picture></div><p></p><h3><strong>The juicy details</strong></h3><p>We test all our endpoints. That includes TCP, TCP w/ TLS, UDP, Pickle, Pickle w/ TLS, and HTTPS. Our test sends a datapoint every second to make sure we pick up on any degradation of service. To make sure we're not falsely covering-up a delay in accepting the data or establishing a connection, the canary has been carefully engineered to send one datapoint per second and to abandon the attempt if it takes longer than a second, ensuring a very reliable continuous rate of one value per second no matter what.</p><p>If it takes longer than a second to connect and submit a datapoint, we'd record that as a dropped datapoint, because we care about the data delivery rate as well as how long it takes for us to accept it.</p><p>A single datapoint gets sent for each protocol and these go to our Graphite-compatible endpoints. The sum rate of these metrics is then rendered in our alerting service and if the value drops below acceptable loss for a particular service, an on-call engineer is paged.</p><p></p><div class="w-embed"><picture>
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second/canaries2.webp" type="image/webp">
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second/canaries2.jpg" type="image/jpeg">
|
||
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second/canaries2.webp" alt="A screenshot of a Hosted Graphite dashboard panel monitoring Canary sumrates">
|
||
<figcaption>Canary sumrates dashboard -- we offset the line at 2.0 from 1.0 for a clear view of both on the same graph</figcaption>
|
||
</picture></div><p></p><h3>Locations</h3><p>Our canary services are located in three places:</p><h4>External</h4><p>The external canaries are all located outside of the main Hosted Graphite architecture/pipeline. They’re tagged and sent from different places, all over the world. We do this so that we can understand if, for example, there’s a problem with traffic from a specific region.</p><h4>Internal</h4><p>We run more canaries that sit within our own data centre. This allows us to test our local network so we can differentiate a failure that is likely due to some mishap in our own service as opposed to external connectivity issues.</p><h4>On-machine</h4><p>While we send canary data from many locations outside and inside our network, we also send a continuous stream of canary data locally from each of our ingestion services to itself so failures of a single machine or service are immediately apparent. This data is also used automatically by our clustering and service discovery tools, allowing auto-remediation of most common individual machine failure types.</p><p></p><div class="w-embed"><picture>
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second/hg3.webp" type="image/webp">
|
||
<source srcset="https://hgblogimg.s3.us-east-2.amazonaws.com/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second/hg3.jpg" type="image/jpeg">
|
||
<img src="https://hgblogimg.s3.us-east-2.amazonaws.com/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second/hg3.webp" alt="A screenshot of Systems Metrics in Hosted Graphite showing "All Systems Operational"">
|
||
<figcaption>Status page (includes full incident history)</figcaption>
|
||
</picture></div><p></p><p>For transparency, <a href="https://status.hostedgraphite.com">we publish all the details on our status page</a> so our customers have full visibility of any blip in our service and real time status information. We include a full breakdown of what caused the issue, who it affected, and how the issue has been resolved.</p></div><div class="sticky-toc-container"><div class="cta-sidebar"><div class="text-block-31"><strong class="bold-text-15">Try Hosted Graphite now!</strong></div><p class="paragraph-12">Get Hosted Graphite free for 14 days. No credit card required.</p><a href="https://www.hostedgraphite.com/accounts/signup/?from=heroku_blogpost" class="cta-side-button w-button">Get Started</a></div><div class="toc w-embed w-script"><script>
|
||
document.addEventListener("DOMContentLoaded", function() {
|
||
function createSideToc(containerId) {
|
||
const container = document.querySelector(containerId);
|
||
|
||
// Create and append the title <h2>
|
||
const tocTitle = document.createElement('h2');
|
||
tocTitle.textContent = 'Table of Contents';
|
||
container.appendChild(tocTitle);
|
||
|
||
const toc = document.createElement('ul');
|
||
toc.className = 'toc';
|
||
|
||
const headers = document.querySelectorAll('.blog-post h2, .blog-post h3, .blog-post h4, .blog-post h5, .blog-post h6');
|
||
const ids = new Set(); // To ensure unique IDs
|
||
|
||
headers.forEach((header) => {
|
||
const id = header.textContent.trim().toLowerCase().replace(/[\s+]+/g, '-').replace(/[^\w\-]+/g, '');
|
||
header.id = id;
|
||
if (!ids.has(id)) {
|
||
ids.add(id);
|
||
|
||
const li = document.createElement('li');
|
||
const a = document.createElement('a');
|
||
a.href = `#${id}`;
|
||
a.textContent = header.textContent;
|
||
li.appendChild(a);
|
||
|
||
toc.appendChild(li);
|
||
}
|
||
});
|
||
|
||
container.appendChild(toc);
|
||
|
||
// Smooth scroll for the new TOC
|
||
toc.querySelectorAll('a').forEach(anchor => {
|
||
anchor.addEventListener('click', function(e) {
|
||
e.preventDefault();
|
||
document.querySelector(this.getAttribute('href')).scrollIntoView({
|
||
behavior: 'smooth'
|
||
});
|
||
history.pushState(null, null, this.getAttribute('href'));
|
||
});
|
||
});
|
||
}
|
||
|
||
// Create the sticky TOC in the sidebar
|
||
createSideToc('.sticky-toc-container');
|
||
});
|
||
</script>
|
||
|
||
<style>
|
||
/* Specific Styling for Sticky TOC */
|
||
.sticky-toc-container {
|
||
position: sticky;
|
||
top: 110px;
|
||
width: 340px;
|
||
height: 100%;
|
||
overflow-y: auto;
|
||
padding: 15px;
|
||
border-radius: 8px;
|
||
font-family: Arial, sans-serif;
|
||
font-size: 14px;
|
||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
|
||
text-align: left;
|
||
}
|
||
|
||
.sticky-toc-container h2 {
|
||
margin-top: 0;
|
||
font-size: 18px;
|
||
color: #2e2e2e;
|
||
}
|
||
|
||
.sticky-toc-container ul {
|
||
list-style-type: none;
|
||
padding-left: 0;
|
||
margin: 0;
|
||
}
|
||
|
||
.sticky-toc-container li {
|
||
margin-bottom: 0.6em;
|
||
}
|
||
|
||
.sticky-toc-container a {
|
||
text-decoration: none;
|
||
color: #8e8b8b;
|
||
}
|
||
|
||
.sticky-toc-container a:hover {
|
||
text-decoration: underline;
|
||
}
|
||
|
||
/* Make sure the TOC has a scroll bar when needed */
|
||
.toc {
|
||
max-height: 35vh;
|
||
overflow-y: auto;
|
||
}
|
||
|
||
/* Media Query to Hide sticky-toc-container on smaller screens */
|
||
@media (max-width: 1215px) {
|
||
.sticky-toc-container {
|
||
display: none;
|
||
}
|
||
}
|
||
</style></div></div></div><div class="w-condition-invisible w-embed w-script"><script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js"></script>
|
||
<pre class="prettyprint">
|
||
|
||
</pre></div><div class="div-block-2"><div class="full-divide"></div><div class="author-wrapper"><a href="/blog-authors/dan-fox" data-w-expand="authors" class="author-name">Dan Fox</a><div class="smallest-divider"></div><div data-w-expand="authors" class="author-bio w-richtext"><p>SRE at Hosted Graphite.</p></div><div><a href="https://twitter.com/hostedgraphite" data-w-expand="authors" class="social-link w-inline-block"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a57aa7ad1fa2300015ca29a_social-18.svg" width="68" alt="" class="image-2"/></a><a data-w-expand="authors" href="#" class="social-link w-inline-block"></a><a href="#" data-w-expand="authors" class="social-link w-inline-block"></a></div></div></div></div></div><div class="main-section gray"><div class="w-container"><div class="section-heading"><h2 class="heading-6">Related Posts</h2><div class="med-divider"></div></div><div class="w-dyn-list"><div role="list" class="w-clearfix w-dyn-items w-row"><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/hosted-graphite-isnt-graphite" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5dd37e9fe192cd2ea5b8af68_Screen%20Shot%202019-11-19%20at%202.14.02%20PM.jpg")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">HostedGraphite</div><div class="preview-text">Hosted Graphite improves upon standard Graphite. Take a look at how we do this and how Hosted Graphite gives your company better functionality. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Shevaun Frazier</div><div class="thumbnail-date">Nov 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/new-developer-onboarding" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5bfea01e7ffa1b30f88f19c0_Blog-background-organge.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">New Developer Onboarding</div><div class="preview-text">Discussions about onboarding tend to revolve around new hires, for obvious reasons, but the process is important for everyone: while the new developer learns the most important aspects of team and company culture, the team has an opportunity to learn new ideas from a fresh pair of eyes. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Heather Wiencko</div><div class="thumbnail-date">Sep 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/pug-life-how-we-run-a-grafana-instance-for-each-user-with-docker" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5d643d90b1c59597a144484a_pug-header.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">PUG Life: How we run a Grafana instance for each user with Docker </div><div class="preview-text">The story of Hosted Grafana at Hosted Graphite, and how we run multiple instances of Grafana. Start monitoring with Hosted Graphite and use our Grafana dashboards directly in-platform. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciarán Finn</div><div class="thumbnail-date">Nov 2020</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/incident-postmortem-template" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Our incident postmortem template</div><div class="preview-text">Sharing our incident postmortem template with some pointers on the review process, what to include in each section, and best practice examples.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Aug 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/its-dead-jim-how-we-write-an-incident-postmortem" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2eed2af56e21241549150_SRE-background.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">"It's dead, Jim": How we write an incident postmortem</div><div class="preview-text">How to write an incident postmortem–what it is, why it’s important, who should write it, and considerations to keep in mind before putting pen to paper.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Fran Garcia</div><div class="thumbnail-date">Jul 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/surviving-on-call-tips-from-a-hosted-graphite-sre" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c48a4405b91aaf448087677_daveblog-header.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Surviving On-Call: Tips from a Hosted Graphite SRE</div><div class="preview-text">Get an insight into what Hosted Graphite's SRE Dave Fennell has learned when being On-Call. Read the tips that he has to offer to make the On-Call experience a positive one.</div></div><div class="thumb-details w-clearfix"><div class="author-title">Dave Fennell</div><div class="thumbnail-date">Jan 2019</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/a-victim-of-its-own-popularity-scaling-our-cloudwatch-integration" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5c095f30e4eab15fb2aa01b4_hg-blog-background-%5BRecovered%5D.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">A victim of its own popularity: Scaling our CloudWatch integration</div><div class="preview-text">Learn how Hosted Graphite scaled its AWS CloudWatch integration. From what to do, how to do it, and deploying it into production. </div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Egan</div><div class="thumbnail-date">Dec 2018</div></div></a></div><div role="listitem" class="blog-thumbnail w-dyn-item w-col w-col-3"><a href="/blog/deadlines-lies-and-videotape-the-tale-of-a-grpc-bug" data-ix="blog-thumbnail" class="thumbnail-wrapper w-inline-block"><div class="image-wrapper"><div style="background-image:url("https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5be2efc2d58fb0c5bfe2fd06_melt66666.png")" class="thumbnail-image"></div><div class="category-tag">Engineering</div></div><div class="thumbnail-text ellipsis"><div class="blog-title">Deadlines lies and videotape: The tale of a gRPC bug</div><div class="preview-text">If you use gRPC in your services, you’ll want to make sure you set a reasonable deadline for your RPC calls, upgrading to gRPC 1.16 as soon as possible is highly recommended. You should also enable client-side keepalive, and adjust the kernel setting for tcp_syn_retries (at least until the fix for this issue gets released).</div></div><div class="thumb-details w-clearfix"><div class="author-title">Ciaran Gaffney</div><div class="thumbnail-date">Nov 2020</div></div></a></div></div></div></div></div><div class="main-section dark"><div class="container-13 w-container"><div class="section-heading"><h2 class="white">See why thousands of engineers trust Hosted Graphite with their monitoring</h2><div class="med-divider"></div></div><a href="https://www.hostedgraphite.com/accounts/signup/" class="get-started-ppc-button w-button">START A FREE TRIAL</a><div class="div-block-14 w-hidden-medium w-hidden-small w-hidden-tiny"><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png" width="120" sizes="(max-width: 991px) 100vw, 120px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity-p-1600.png 1600w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd163a0eb5000019d5082_xfinity.png 2000w" alt="" class="image-5"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png" width="190" sizes="(max-width: 991px) 100vw, 190px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-800.png 800w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo-p-1080.png 1080w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd1603e2b760001e13b08_Gov_uk_logo.png 1280w" alt="" class="image-3"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1)-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd15f3e2b760001e13b07_playtika_logo%20(1).png 620w" alt="" class="image-4"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png" width="180" sizes="(max-width: 991px) 100vw, 180px" srcset="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo-p-500.png 500w, https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd4056b273e0001a3d7d7_atlassian_logo.png 800w" alt="" class="image-6"/><img src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a5cd52f6b273e0001a3d86c_Tableau%20(1).svg" alt="" class="image-7"/></div></div><div class="footer-section"><div class="w-container"><div>© 2024 Metricfire Limited. <br/>All Rights Reserved.<br/><br/><br/>5940 S Rainbow Blvd Ste 400<br/>Las Vegas, NV 89118-2507<br/>United States</div></div></div></div><div class="w-embed"></div><div class="w-embed w-script"><script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "Article",
|
||
"mainEntityOfPage": {
|
||
"@type": "WebPage",
|
||
"@id": "https://www.hostedgraphite.com/blog/continuous-self-testing-at-hosted-graphite-why-we-send-external-canaries-every-second"
|
||
},
|
||
"headline": "Continuous self-testing at Hosted Graphite — why we send external canaries, every second",
|
||
"image": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad08c3e0e97b8494f0194b4_img_20170705_150424_1024-3.jpg",
|
||
"thumbnailUrl": "https://cdn.prod.website-files.com/5a57aa7ad1fa2300015ca257/5ad08c3e0e97b8494f0194b4_img_20170705_150424_1024-3.jpg",
|
||
"author": {
|
||
"@type": "Person",
|
||
"name": "Dan Fox",
|
||
"url": "dan-fox"
|
||
},
|
||
"publisher": {
|
||
"@type": "Organization",
|
||
"name": "Metricfire Limited",
|
||
"logo": {
|
||
"@type": "ImageObject",
|
||
"url": "https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/5a589d3f3c31ba0001f33fca_Orange%20HG%20logo.svg"
|
||
}
|
||
},
|
||
"datePublished": "Jul 06, 2017",
|
||
"dateModified": "Sep 28, 2021",
|
||
"description": "At Hosted Graphite, it is of utmost importance to monitor our internal systems. Learn more on how our engineers use different types of canaries for continuous self-testing."
|
||
}
|
||
</script>
|
||
|
||
<script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "SoftwareApplication",
|
||
"applicationCategory": "BusinessApplication",
|
||
"name": "Hosted Graphite",
|
||
"description": "Hosted Graphite offers powerful monitoring and alerting for various business applications, providing integrations with multiple data sources, a Grafana UI, and great customer support.",
|
||
"review": [
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Cecily G." },
|
||
"reviewBody": "This system is the tool every engineer needs to be successful. Easy to use and captures metrics that no other system has successfully been able to pull for us.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Christopher C." },
|
||
"reviewBody": "Great product suite, outstanding customer service! Running a distributed data pipeline is challenging to keep healthy, and Hosted Graphite has been an instrumental partner for monitoring system health and getting ahead of issues.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "UC" },
|
||
"reviewBody": "A great service with excellent support. Simple to use, inexpensive, and great support.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Brendan C." },
|
||
"reviewBody": "High-quality metrics hosting. Hosted Graphite enables us to instrument our products and processes efficiently. We can proactively monitor for issues and use dashboards to identify more minor performance problems before they become major ones.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Jacobson M." },
|
||
"reviewBody": "Perfect for remote device analytics. Hosted graphite makes it easy for me to keep track of my remote devices being used by my customers. I can't always be on site, so this allows easy tracking of device details.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 }
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "James C." },
|
||
"reviewBody": "The easiest way for DevOps to incorporate Kubernetes without having to suffer a paradigm shift away from using Graphite. Prometheus, while academically good, forces security worst practices which were unacceptable for us as a SOC2 compliant organization. Graphite is battle-proven and already integrated, so Hosted Graphite allowed us to do what we needed without drastic changes.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
|
||
"datePublished": "2021-03-19"
|
||
},
|
||
{
|
||
"@type": "Review",
|
||
"author": { "@type": "Person", "name": "Anne M." },
|
||
"reviewBody": "Offers great hosting service, Hosted Graphite is quite easy to use and highly secure. I like how they offer the trial version to evaluate if the solution fits your needs. In terms of setup, it's easy with minimal registration details and flexible pricing.",
|
||
"reviewRating": { "@type": "Rating", "bestRating": 5, "ratingValue": 5, "worstRating": 1 },
|
||
"datePublished": "2019-02-20"
|
||
}
|
||
],
|
||
"operatingSystem": "All major operating systems",
|
||
"aggregateRating": {
|
||
"@type": "AggregateRating",
|
||
"reviewCount": 7,
|
||
"ratingValue": 4.85,
|
||
"bestRating": 5,
|
||
"worstRating": 1
|
||
},
|
||
"offers": {
|
||
"@type": "Offer",
|
||
"price": 19.00,
|
||
"priceCurrency": "USD",
|
||
"availability": "https://schema.org/InStock",
|
||
"url": "https://hostedgraphite.com",
|
||
"seller": {
|
||
"@type": "Organization",
|
||
"name": "Hosted Graphite"
|
||
}
|
||
}
|
||
}
|
||
</script></div><script src="https://d3e54v103j8qbb.cloudfront.net/js/jquery-3.5.1.min.dc5e7f18c8.js?site=5a57aa76d1fa2300015ca244" type="text/javascript" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.36b8fb49256177c8.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.schunk.83de43c5e9eda212.js" type="text/javascript"></script><script src="https://cdn.prod.website-files.com/5a57aa76d1fa2300015ca244/js/webflow.54e1a7f0.3d4248ede2db61fe.js" type="text/javascript"></script><!-- Google Tag Manager (noscript) -->
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-NT79GSP"
|
||
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
<!-- End Google Tag Manager (noscript) --><script>
|
||
img = document.querySelectorAll('.w-embed');
|
||
img.forEach(element => element.style.textAlign = 'center');
|
||
</script></body></html> |