680 lines
43 KiB
HTML
680 lines
43 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en" dir="ltr" prefix="og: https://ogp.me/ns#">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="description" content="In the first of a new series focusing on DevOps monitoring tools, we review the various types of system observability tools.">
|
|
<link rel="canonical" href="https://opensource.com/article/18/8/now-available-open-source-guide-devops-monitoring-tools">
|
|
<link rel="image_src" href="https://opensource.com/sites/default/files/lead-images/devops_monitoring.png">
|
|
<meta name="author" content="Dan Barker">
|
|
<meta property="og:site_name" content="Opensource.com">
|
|
<meta property="og:url" content="https://opensource.com/article/18/8/now-available-open-source-guide-devops-monitoring-tools">
|
|
<meta property="og:title" content="The open source guide to DevOps monitoring tools">
|
|
<meta property="og:description" content="In the first of a new series focusing on DevOps monitoring tools, we review the various types of system observability tools.">
|
|
<meta property="og:image" content="https://opensource.com/sites/default/files/lead-images/devops_monitoring.png">
|
|
<meta name="twitter:card" content="summary">
|
|
<meta name="twitter:site" content="@opensourceway">
|
|
<meta name="twitter:description" content="In the first of a new series focusing on DevOps monitoring tools, we review the various types of system observability tools.">
|
|
<meta name="twitter:title" content="The open source guide to DevOps monitoring tools">
|
|
<meta name="twitter:creator" content="@barkerd427">
|
|
<meta name="twitter:image" content="https://opensource.com/sites/default/files/lead-images/devops_monitoring.png">
|
|
<meta name="Generator" content="Drupal 10 (https://www.drupal.org)">
|
|
<meta name="MobileOptimized" content="width">
|
|
<meta name="HandheldFriendly" content="true">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<link rel="icon" href="/themes/osdc/assets/img/favicons/favicon.ico" type="image/vnd.microsoft.icon">
|
|
|
|
<title>The open source guide to DevOps monitoring tools | Opensource.com</title>
|
|
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_v1Pk9uCDi2UfNhTAnRmgqTOT8FMeyU3TJ1MbtJc3rko.css?delta=0&language=en&theme=osdc&include=eJxVj9FuwzAIRX_Irj9pIviusYpNBLhV9vXLmuyhL-joCg6wqUipNjeS23ZwljYennz3QC8LORILue8f3OFOd3hiNZSh1knaD5J65XIXXUhO_lOeRDNWtdwRlIR2nfFVm7M-YXvRAdarkYPyIsqPbPg-I4NQoGbWERiRng0vL-9661qn4GPHe_pKLBoLcuC43pIdmvLKsc6--Nz-3xlaL4ODjNdftORqDw">
|
|
<link rel="stylesheet" media="all" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
|
|
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_pwV-EcCwseDS_ZsbU4fuelxCUlfePETEO0FOdvVn9o8.css?delta=2&language=en&theme=osdc&include=eJxVj9FuwzAIRX_Irj9pIviusYpNBLhV9vXLmuyhL-joCg6wqUipNjeS23ZwljYennz3QC8LORILue8f3OFOd3hiNZSh1knaD5J65XIXXUhO_lOeRDNWtdwRlIR2nfFVm7M-YXvRAdarkYPyIsqPbPg-I4NQoGbWERiRng0vL-9661qn4GPHe_pKLBoLcuC43pIdmvLKsc6--Nz-3xlaL4ODjNdftORqDw">
|
|
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_OxBWfCNbQ0DfjsXj7OrV5J5bVQ9sXQvcpEDcyMq1-js.css?delta=3&language=en&theme=osdc&include=eJxVj9FuwzAIRX_Irj9pIviusYpNBLhV9vXLmuyhL-joCg6wqUipNjeS23ZwljYennz3QC8LORILue8f3OFOd3hiNZSh1knaD5J65XIXXUhO_lOeRDNWtdwRlIR2nfFVm7M-YXvRAdarkYPyIsqPbPg-I4NQoGbWERiRng0vL-9661qn4GPHe_pKLBoLcuC43pIdmvLKsc6--Nz-3xlaL4ODjNdftORqDw">
|
|
<link rel="stylesheet" media="all" href="https://fonts.googleapis.com/css2?family=Red+Hat+Text:wght@400;500;700&display=swap">
|
|
<link rel="stylesheet" media="all" href="https://fonts.googleapis.com/css2?family=Red+Hat+Mono:wght@400;500;700&display=swap">
|
|
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_420Pl5I2-T-N-GwE2OsyRKPNkmjkwR63uikCNuQS2Pw.css?delta=6&language=en&theme=osdc&include=eJxVj9FuwzAIRX_Irj9pIviusYpNBLhV9vXLmuyhL-joCg6wqUipNjeS23ZwljYennz3QC8LORILue8f3OFOd3hiNZSh1knaD5J65XIXXUhO_lOeRDNWtdwRlIR2nfFVm7M-YXvRAdarkYPyIsqPbPg-I4NQoGbWERiRng0vL-9661qn4GPHe_pKLBoLcuC43pIdmvLKsc6--Nz-3xlaL4ODjNdftORqDw">
|
|
|
|
<script src="/libraries/chapeaux--cpx-reporter/cpx-reporter.js?v=0.12.0" type="module"></script>
|
|
<script src="https://www.redhat.com/ma/dpal.js"></script>
|
|
|
|
<script async data-debug data-domain="redhat2.com" id="trustarc" src="https://static.redhat.com/libs/redhat/marketing/latest/trustarc/trustarc.js" type="text/javascript"></script>
|
|
|
|
<script type="module" src="/themes/osdc/node_modules/@patternfly/pfe-modal/dist/pfe-modal.min.js"></script>
|
|
</head>
|
|
<body class="path-node page-node-type-article">
|
|
<a href="#main-content" class="focusable skip-link">
|
|
Skip to main content
|
|
</a>
|
|
|
|
<div class="dialog-off-canvas-main-canvas" data-off-canvas-main-canvas>
|
|
<div class="layout-container">
|
|
|
|
<header>
|
|
<div class="region region-messages">
|
|
<div data-drupal-messages-fallback class="hidden"></div>
|
|
|
|
</div>
|
|
|
|
|
|
<div class="region-top-bar">
|
|
<div class="region wrapper region-user-menu">
|
|
<div class="hamburger__container">
|
|
<div class="hamburger">
|
|
<button id="mobnav-btn" aria-label="Toggle main menu" class="button--plain toggle-menu"></button>
|
|
<div class="bar"></div>
|
|
</div>
|
|
</div>
|
|
|
|
<a href="https://www.redhat.com/en?intcmp=701600000011l7VAAQ" target="_blank" class="flex-center support-logo">
|
|
<img alt="Supported by Red Hat" src="/themes/osdc/assets/img/l_supported-by-redhat-white.svg" title="Supported by Red Hat" width="125" height="56" class="support-logo__image">
|
|
</a>
|
|
|
|
<button id="toggle-search" aria-label="Toggle Search" class="button--plain toggle-search">
|
|
<i class="icon-search"></i>
|
|
</button>
|
|
<nav role="navigation" aria-labelledby="block-useraccountmenu-menu" id="block-useraccountmenu" class="block block-menu navigation menu--account">
|
|
|
|
<h2 class="visually-hidden" id="block-useraccountmenu-menu">User account menu</h2>
|
|
|
|
|
|
|
|
|
|
<ul class="account">
|
|
|
|
<li class="log-in account__item">
|
|
<a href="/user/login?current=/article/18/8/now-available-open-source-guide-devops-monitoring-tools" class="account__link" data-drupal-link-system-path="user/login">Log in</a>
|
|
</li>
|
|
<li class="rss account__item">
|
|
<a href="/feed" title="RSS" class="account__link" data-drupal-link-system-path="rss.xml">RSS</a>
|
|
</li>
|
|
</ul>
|
|
|
|
|
|
|
|
</nav>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
<div class="region region-header">
|
|
<div id="block-osdc-branding" class="block block-system block-system-branding-block">
|
|
|
|
|
|
<div class="branding">
|
|
<a href="/" rel="home" aria-label="Opensource.com" class="branding__logo">
|
|
<img src="/themes/osdc/logo.svg" alt="Home">
|
|
</a>
|
|
<a href="https://www.redhat.com/en?intcmp=701600000011l7VAAQ" target="_blank" class="branding__supported-by">
|
|
<img src="/themes/osdc/assets/img/l_supported-by-redhat-white.svg" width="125" height="56" class="branding__supported-by__image" alt="Supported by Red Hat">
|
|
</a>
|
|
</div>
|
|
<div class="branding__slogan"></div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</header>
|
|
|
|
<div class="region region-main-nav">
|
|
<nav role="navigation" aria-labelledby="block-mainnavigation-menu" id="block-mainnavigation" class="block block-menu navigation menu--main">
|
|
|
|
<h2 class="visually-hidden" id="block-mainnavigation-menu">Main navigation</h2>
|
|
|
|
|
|
|
|
|
|
<ul class="main" id="main-menu">
|
|
|
|
<li class="main__item">
|
|
<a href="/" class="main__link" data-drupal-link-system-path="<front>">Articles</a>
|
|
</li>
|
|
<li class="main__item is-expanded">
|
|
<a href="/resources" class="main__link" data-drupal-link-system-path="node/17752">Resources</a>
|
|
|
|
<ul class="main__submenu">
|
|
|
|
<li class="main__item is-expanded">
|
|
<a href="/resources/what-open-source" class="main__link" data-drupal-link-system-path="node/17753">What is open source?</a>
|
|
|
|
<ul class="main__submenu">
|
|
|
|
<li class="main__item">
|
|
<a href="/open-source-way" class="main__link" data-drupal-link-system-path="node/17730">The open source way</a>
|
|
</li>
|
|
</ul>
|
|
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/resources/projects-and-applications" class="main__link" data-drupal-link-system-path="node/17760">Projects and applications</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/resources/organizations" class="main__link" data-drupal-link-system-path="node/17763">Organizations</a>
|
|
</li>
|
|
<li class="main__item is-expanded">
|
|
<a href="/alternatives" class="main__link" data-drupal-link-system-path="node/30871">Open source alternatives</a>
|
|
|
|
<ul class="main__submenu">
|
|
|
|
<li class="main__item">
|
|
<a href="/alternatives/adobe-acrobat" class="main__link" data-drupal-link-system-path="node/29486">Alternatives to Acrobat</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/autocad" class="main__link" data-drupal-link-system-path="node/29001">Alternatives to AutoCAD</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/dreamweaver" class="main__link" data-drupal-link-system-path="node/27646">Alternatives to Dreamweaver</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/gmail" class="main__link" data-drupal-link-system-path="node/22941">Alternatives to Gmail</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/matlab" class="main__link" data-drupal-link-system-path="node/29276">Alternatives to MATLAB</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/minecraft" class="main__link" data-drupal-link-system-path="node/22786">Alternatives to Minecraft</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/google-photos" class="main__link" data-drupal-link-system-path="node/27211">Alternatives to Google Photos</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/life/12/6/design-without-debt-five-tools-for-designers" class="main__link" data-drupal-link-system-path="node/16365">Alternatives to Photoshop</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/skype" class="main__link" data-drupal-link-system-path="node/45576">Alternatives to Skype</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/slack" class="main__link" data-drupal-link-system-path="node/23286">Alternatives to Slack</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/trello" class="main__link" data-drupal-link-system-path="node/22546">Alternatives to Trello</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives" class="main__link" data-drupal-link-system-path="node/30871">More...</a>
|
|
</li>
|
|
</ul>
|
|
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/resources/linux" class="main__link" data-drupal-link-system-path="node/19796">Linux</a>
|
|
</li>
|
|
</ul>
|
|
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/downloads" class="main__link" data-drupal-link-system-path="downloads">Downloads</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/faq" class="main__link" data-drupal-link-system-path="node/62466">Frequently Asked Questions</a>
|
|
</li>
|
|
</ul>
|
|
|
|
|
|
|
|
</nav>
|
|
<div class="views-exposed-form block block-views block-views-exposed-filter-blockacquia-search-page" data-drupal-selector="views-exposed-form-acquia-search-page" id="block-exposedformacquia-searchpage">
|
|
|
|
|
|
<form action="/site-search" method="get" id="views-exposed-form-acquia-search-page" accept-charset="UTF-8">
|
|
<div class="form--inline clearfix">
|
|
<div class="js-form-item form-item js-form-type-textfield form-type-textfield js-form-item-search form-item-search">
|
|
<label for="edit-search">Search</label>
|
|
<input placeholder="Search" data-drupal-selector="edit-search" type="text" id="edit-search" name="search" value size="30" maxlength="128" class="form-text">
|
|
|
|
</div>
|
|
<div data-drupal-selector="edit-actions" class="form-actions js-form-wrapper form-wrapper" id="edit-actions--2"><input data-drupal-selector="edit-submit-acquia-search" type="submit" id="edit-submit-acquia-search" value="Go" class="button js-form-submit form-submit">
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</form>
|
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
|
|
<main>
|
|
<a id="main-content" tabindex="-1"></a>
|
|
<div class="wrapper wrapper--870">
|
|
<div class="region region-content">
|
|
<div id="block-osdc-page-title" class="block block-core block-page-title-block">
|
|
|
|
|
|
|
|
<h1 class="published page-title">
|
|
<span class="field field--name-title field--type-string field--label-hidden">The open source guide to DevOps monitoring tools</span>
|
|
|
|
</h1>
|
|
|
|
|
|
</div>
|
|
<div id="block-mainpagecontent" class="block block-system block-system-main-block">
|
|
|
|
|
|
|
|
<article class="node node--type-article node--promoted node--view-mode-full">
|
|
|
|
|
|
<div class="article__subhead">In the first of a new series focusing on DevOps monitoring tools, we review the various types of system observability tools.</div>
|
|
|
|
<footer class="author__meta">
|
|
<div class="author__submitted">
|
|
<div class="author__list">
|
|
By
|
|
<article class="profile">
|
|
<a href="/users/barkerd427" hreflang="en">Dan Barker</a>
|
|
<span class="user-attributes">(Alumni)</span>
|
|
|
|
</article>
|
|
|
|
</div>
|
|
<div class="author__meta-date-container">
|
|
<span class="author__meta-date">August 28, 2018</span> <span class="author__date-pipe">|</span>
|
|
<a href="#comments"><span class="author__comment-count">0 Comments </span></a>
|
|
<span class="author__date-pipe">|</span>
|
|
<pfe-readtime class="pf-global--FontSize--lg" for="#article_content" hidden>%t min read</pfe-readtime>
|
|
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
<div id="article_content" class="author__content">
|
|
|
|
<div class="layout layout--onecol">
|
|
<div class="layout__region layout__region--content">
|
|
<div class="block block-layout-builder block-extra-field-blocknodearticlerate-rate-widget">
|
|
|
|
|
|
<div class="rate-widget thumbsup"><form class="node-article-47266-updown-rate-widget-0 vote-form" id="rate-widget-base-form" data-drupal-selector="node-article-47266-updown-rate-widget-0" action="/article/18/8/now-available-open-source-guide-devops-monitoring-tools" method="post" accept-charset="UTF-8">
|
|
<div class="rate-widget-thumbs-up">
|
|
<div class="thumbsup-rating-wrapper rate-disabled" can-edit="false"><div class="form-item js-form-item form-type-radio js-form-type-radio form-item-value js-form-item-value form-disabled">
|
|
<label class="form-item js-form-item form-type-radio js-form-type-radio form-item-value js-form-item-value form-disabled rating-label thumbsup-rating-label thumbsup-rating-label-up vote-pending rate-thumbs-up-btn-up"><input twig-suggestion="rating-input" class="rating-input thumbsup-rating-input form-radio" data-drupal-selector="edit-value-1" disabled="disabled" type="radio" id="edit-value-1" name="value" value="1">
|
|
</label>
|
|
</div>
|
|
</div><input data-drupal-selector="form-3hh8qgftnfcgeg50852ieydpxna-52zomscep8-r8wu" type="hidden" name="form_build_id" value="form-3hH8qGfTNFcGeG50852IEyDpXna_52ZOmsCEP8-r8wU">
|
|
<input data-drupal-selector="edit-node-article-47266-updown-rate-widget-0" type="hidden" name="form_id" value="node_article_47266_updown_rate_widget_0">
|
|
<input class="thumbsup-rating-submit button button--primary js-form-submit form-submit" data-drupal-selector="edit-submit" data-disable-refocus="true" type="submit" id="edit-submit--2" name="op" value="Save">
|
|
<div class="vote-result js-form-wrapper form-wrapper" data-drupal-selector="edit-result" id="edit-result"><div class="rate-widget-thumbs-up rate-score">
|
|
414 readers like this. </div>
|
|
</div>
|
|
|
|
<div class="rate-score"></div>
|
|
</div>
|
|
|
|
|
|
</form>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="block block-layout-builder block-extra-field-blocknodearticlecontent-moderation-control">
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
<div class="layout layout--onecol">
|
|
<div class="layout__region layout__region--content">
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-lead-image">
|
|
|
|
|
|
|
|
<div class="field field--name-field-lead-image field--type-entity-reference field--label-hidden field__item"><article class="media media--type-image media--view-mode-caption">
|
|
|
|
|
|
<div class="field field--name-field-media-image field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/lead-images/devops_monitoring.png" width="520" height="292">
|
|
|
|
</div>
|
|
|
|
<div class="field field--name-field-caption field--type-text-long field--label-hidden caption field__item"><span class="caption__byline">Image by: </span><p>opensource.com</p>
|
|
</div>
|
|
|
|
</article>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="block block-layout-builder block-field-blocknodearticlebody">
|
|
|
|
|
|
|
|
<div class="clearfix text-formatted field field--name-body field--type-text-with-summary field--label-hidden field__item"><p>In this new series, we'll focus on DevOps monitoring and observability tools. Over the next few weeks, we’ll explore metrics aggregation and monitoring, log aggregation, alerting and visualizations, and distributed tracing. Alternatively, you can download the entire <a href="/downloads/devops-monitoring-guide" target="_blank">open source guide to DevOps monitoring tools</a> now.</p>
|
|
<p>Let’s get started.</p>
|
|
<h2>A tale of two views</h2>
|
|
<p>Once upon a time, I was troubleshooting some vexing problems in an application that needed to be scaled several orders of magnitude, with only a couple of weeks to re-architect it. We had no log aggregation, no metrics aggregation, no distributed tracing, and no visualization. Most of our work had to be done on the actual production nodes using tools like <a href="https://en.wikipedia.org/wiki/Strace" target="_blank">strace</a> and grepping through logs. These are great tools, but they don’t make it easy to analyze a distributed system across dozens of hosts. We got the job done, but it was painful and involved a lot more guessing and risk than I prefer.</p>
|
|
<p>At a different job, I helped troubleshoot an app in production that was suffering from an out-of-memory (OOM) issue. The problem was inconsistent, as it didn’t seem to correlate with running time, load, time of day, or any other aspect that would provide some predictability. This was obviously going to be a difficult problem to diagnose on a system that spanned hundreds of hosts with many applications calling it. Luckily, we had log aggregation, distributed tracing, metrics aggregation, and a plethora of visualizations. We looked at our memory graph and saw a distinct spike in memory usage, so we used that spike to alert us so we could diagnose the issue in real time when it occurred.</p>
|
|
<p>When we received an alert, we went to our log aggregation system to correlate the logs to the memory spike. We found the OOM error and the related calls around it. We then understood what application was calling the service that resulted in the spike and used that information to find the exact transaction that caused the issue. We determined that someone had stored a huge file in a database that our service was trying to load, but the service was running out of memory before it could fully load and process the record. We should have been defending against this in the first place, but we were happy to find it so quickly and fix it with very little effort. Once we understood the error, we discovered a lot of records had large files like this, and we didn’t need that part of the record to function properly.</p>
|
|
<p>You might think the second situation happened a long time after the first and we had improved over time. Or maybe you suspect that when I changed jobs, my new company had better tooling. In reality, the second situation happened before the first one. I moved from a company with fairly advanced observability tools to one with no observability tools. It was strikingly disturbing as the developer to have an application in production and know nothing about it. I learned a lot about the importance of system observability and the related tools as I began rebuilding that infrastructure. Also, Mike Julian's <a href="https://www.practicalmonitoring.com/">Practical Monitoring</a> is a must-read for those who want to know more about their systems.</p>
|
|
<div class="embedded-callout-menu callout-float-right">
|
|
<div class="view view-related-content-callout view-id-related_content_callout view-display-id-default">
|
|
<div class="view-header">
|
|
<p>More DevOps resources</p>
|
|
</div>
|
|
<div class="view-content">
|
|
<div class="item-list">
|
|
<ul>
|
|
<li><a href="/resources/devops?intcmp=7013a00000263HlAAI">What is DevOps?</a></li>
|
|
<li><a href="/downloads/devops-hiring-guide?intcmp=7013a00000263HlAAI">The ultimate DevOps hiring guide</a></li>
|
|
<li><a href="/downloads/devops-monitoring-guide?intcmp=7013a00000263HlAAI">DevOps monitoring tools guide</a></li>
|
|
<li><a href="/downloads/devsecops?intcmp=7013a00000263HlAAI">Getting started with DevSecOps</a></li>
|
|
<li><a href="https://enterprisersproject.com/cheat-sheet-devops-glossary?intcmp=7013a00000263HlAAI">Download the DevOps glossary</a></li>
|
|
<li><a href="https://www.ansible.com/resources/ebooks/ansible-for-devops?intcmp=7013a00000263HlAAI">eBook: Ansible for DevOps</a></li>
|
|
<li><a href="/tags/devops?src=devops_resource_menu2">Latest DevOps articles</a></li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<h2>Observability principles</h2>
|
|
<p>So, what are observability tools? Actually, what is observability?</p>
|
|
<p>Observability isn’t just a marketing term; it’s a component of <a href="https://en.wikipedia.org/wiki/Control_theory" target="_blank">control theory</a>. If you want a quick primer, this <a href="https://www.youtube.com/watch?v=iRZmJBcg1ZA" target="_blank">video</a> might be helpful. Basically, observability means that you can estimate a particular state of a system based on an output. More generally, a system’s state should be deterministic from its outputs. Controllability, the <a href="https://en.m.wikipedia.org/wiki/Duality_(mathematics)" target="_blank">mathematical dual</a> of observability, of a system requires that a system state be determined by the inputs to the system.</p>
|
|
<p>This is a fairly simple concept, but it’s very challenging to put into practice. In a sufficiently complex system, it may be nearly impossible to implement full observability. However, you should strive to get the right outputs that allow you to determine the system’s state, especially when you encounter a failure mode.</p>
|
|
<h2>Observability tool types</h2>
|
|
<p>In this series, we’ll dig into different types of observability tools. For each type, we’ll cover what they’re used for, what specific tools are available, some use cases, and any unique characteristics that may come up during your search for a new tool. These are presented in the order you should implement them. Metrics aggregation is first, as it’s often easy to instrument an application built with any modern language. Second is logging because it will require more application modifications but provides tremendous value. Third is alerting and visualizations, which require the first two types for full functionality. And last is distributed tracing, as it may not be necessary in a simple monolith and is much harder to implement fully.</p>
|
|
<h3>Metrics aggregation</h3>
|
|
<p>This type of tool generally consists of time-series data. Time-series data is time-ordered data, and it is normally collected with an internally consistent interval. This consistency allows some advanced calculations to be applied to the series and provides for predictive analytics using simple regressions or more advanced algorithms.</p>
|
|
<h3>Log aggregation</h3>
|
|
<p>These tools deal with data types that are related more to events than to a series of consistent data points. This output is often emitted as a system enters some undesired state. Some systems output a lot of logs that don’t fit this condition. We’ll cover more of the do’s and don’ts of logging in a future article.</p>
|
|
<h3>Alerting/visualizations</h3>
|
|
<p>This may not appear to fit with the other types listed, as it’s really subsequent to the others, but it provides a consumable output for the other types and can produce its own outputs. These types of tools generally make the system more understandable to humans. They also help create a more interactive system through both proactive and reactive notifications about negative system states.</p>
|
|
<h3>Distributed tracing</h3>
|
|
<p>Much like tracing within a single application, distributed tracing allows you to follow a single transaction through an entire system. This allows you to home in on specific transactions that might be experiencing problems. Due to performance concerns, a sampling algorithm is often applied.</p>
|
|
<h2>Common DevOps features</h2>
|
|
<p>There are several aspects you should look for in any type of observability tool. We’ll cover these generally now and will bring them back up in future articles.</p>
|
|
<h3>OpenAPI</h3>
|
|
<p>This specification was previously called Swagger but renamed when it was adopted by the <a href="https://www.openapis.org/" target="_blank">OpenAPI Initiative</a> within the <a href="https://training.linuxfoundation.org/" target="_blank">Linux Foundation</a>. The OpenAPI Specification is a language-agnostic tool that can automatically generate documentation of methods, parameters, and models. This is commonly used to generate RESTful interfaces in HTTP, but it is also protocol-agnostic. A user can create a client in almost any language if one doesn't already exist. Every tool should have this type of API (or should be getting it soon). If your tool doesn’t have it yet, you may want to look elsewhere. Tools that haven't implemented this specification or don't have it on their roadmap likely have other deficiencies in adopting open, modern standards and code.</p>
|
|
<h3>Open source</h3>
|
|
<p>There are a lot of good tools in this space that aren’t open source but may be the right fit for your company. If you pick one of those tools, make sure its documentation and accessory tooling are open source. Open source observability tools can provide valuable insights into how your other observability tools are functioning (or maybe not functioning). They also offer all the other benefits of any open source project which you can read more about on <a href="/" target="_blank">Opensource.com</a>.</p>
|
|
<h3>Open standards</h3>
|
|
<p>Regardless of whether or not a tool is open source, it should always use open standards when possible. We’ve already discussed one of these, OpenAPI, but there are many more. We’ll discuss these standards in the appropriate sections to ensure you know they exist and where they’re used.</p>
|
|
<h3>Wide dissemination</h3>
|
|
<p>Part of observability and openness is allowing everyone to view data. The tools you pick should be open by default. You may want to restrict some areas, but you’ll want to default to open and close access only if it’s absolutely required. You never know who in your company might want to solve your problem or who you’ll need to bring in to help solve a problem. The last thing you’ll want is access barriers when troubleshooting your income source.</p>
|
|
<h3>Federated model (preferred)</h3>
|
|
<p>This is similar to defaulting to open, but it allows everyone to provide input and control their own areas more locally. Many legacy systems are architected in a way that requires all data to flow through a central system regardless of need. This also centralizes control around that data. A federated system allows for local aggregation, processing, and control while allowing a central organization to collect the same data or summarized data. The central system likely only wants a subset of the data stored at the local level. This model increases agility, flexibility, and usability.</p>
|
|
<p>In this series, we’ll be exploring each of the observability tool types in more detail. We’ll also help you choose the right tool for your use case. Feel free to read them in any order you want—or you can <a href="/downloads/devops-monitoring-guide" target="_blank">download the entire guide</a>.</p>
|
|
<p> </p>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-listicle-title">
|
|
|
|
|
|
|
|
<div class="field field--name-field-listicle-title field--type-string field--label-hidden field__item">What to read next</div>
|
|
|
|
</div>
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-listicles">
|
|
|
|
|
|
|
|
<div class="field field--name-field-listicles field--type-entity-reference field--label-hidden field__items">
|
|
<div class="field__item"><article class="teaser">
|
|
<figure>
|
|
<div class="field field--name-field-lead-image field--type-entity-reference field--label-hidden field__item"> <a href="/article/18/8/open-source-monitoring-tools" hreflang="und"><img loading="lazy" src="/sites/default/files/styles/article_teaser/public/lead-images/mistake_bug_fix_find_error.png?itok=L2RGRyRm" width="360" height="202" alt="magnifying glass on computer screen, finding a bug in the code" title="magnifying glass on computer screen, finding a bug in the code" class="image-style-article-teaser">
|
|
|
|
</a>
|
|
</div>
|
|
</figure>
|
|
<div class="teaser__content">
|
|
|
|
<h2>
|
|
<a href="/article/18/8/open-source-monitoring-tools" rel="bookmark"><span class="field field--name-title field--type-string field--label-hidden">4 open source monitoring tools</span>
|
|
</a>
|
|
</h2>
|
|
|
|
|
|
<div class="clearfix text-formatted field field--name-field-article-subhead field--type-text-long field--label-hidden field__item"> <p>Here's what you need to know about time-series data and metrics aggregation tools.</p>
|
|
|
|
|
|
</div>
|
|
|
|
<footer>
|
|
<div class="teaser__author">
|
|
|
|
<div class="user__item">
|
|
<pfe-avatar name="Dan Barker" shape="circle" pattern="squares" src="/sites/default/files/pictures/me_14.jpg" pfelement class="PFElement"></pfe-avatar>
|
|
<div class="user__name">
|
|
<a href="/users/barkerd427" hreflang="en">Dan Barker</a>
|
|
</div>
|
|
<span class="user__attributes"><span class="user-attributes">(Alumni)</span>
|
|
</span>
|
|
</div>
|
|
|
|
</div>
|
|
<span class="teaser__date">August 29, 2018</span>
|
|
</footer>
|
|
</div>
|
|
</article>
|
|
|
|
|
|
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-tags">
|
|
|
|
|
|
|
|
<div class="field field--name-field-tags field--type-entity-reference field--label-inline clearfix">
|
|
<div class="field__label">Tags</div>
|
|
<div class="field__items">
|
|
<div class="field__item"><a href="/tags/devops" hreflang="en">DevOps</a></div>
|
|
<div class="field__item"><a href="/tags/sysadmin" hreflang="en">Sysadmin</a></div>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="views-element-container author-block block block-views block-views-blockauthor-info-author-info">
|
|
<div class="author-block__wrapper">
|
|
|
|
|
|
<div><div class="view view-author-info view-id-author_info view-display-id-author_info js-view-dom-id-cfad5e490651c710d18c27eca5afacee2cada506a5b73eedb44e02c5989b32f0">
|
|
|
|
|
|
|
|
<div class="view-content">
|
|
<div class="views-row"><div class="views-field views-field-field-display-name"><div class="field-content"><a href="/users/barkerd427" hreflang="en">Dan Barker</a></div></div><div class="views-field views-field-field-user-picture"><div class="field-content"><article class="media media--type-image media--view-mode-_50x150">
|
|
|
|
|
|
<div class="field field--name-field-media-image field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/styles/150x150/public/pictures/me_14.jpg?itok=FsZ7k9nA" width="150" height="150" alt="Dan Barker" class="image-style-_50x150">
|
|
|
|
|
|
</div>
|
|
|
|
</article>
|
|
</div></div><div class="views-field views-field-field-user-about"><span class="field-content">Website: https://danbarker.codes
|
|
Email: dan@danbarker.codes</span></div><div class="views-field views-field-nothing"><span class="field-content"><a href="/users/barkerd427">More about me</a></span></div></div>
|
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="views-element-container author-block block block-views block-views-blockauthor-info-block-1">
|
|
<div class="author-block__wrapper plain">
|
|
|
|
|
|
<div><div class="view view-author-info view-id-author_info view-display-id-block_1 js-view-dom-id-9e428b28e0a4c61b81099bf0da36e4e39c9f04d21201da32be186b3bd0e53c29">
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-comments">
|
|
|
|
|
|
<section id="comments" class="field field--name-field-comments field--type-comment field--label-hidden comment-wrapper">
|
|
|
|
|
|
<div class="comments__count">
|
|
<h2>Comments are closed.</h2>
|
|
<div class="login">These comments are closed.</div>
|
|
</div>
|
|
|
|
|
|
|
|
</section>
|
|
|
|
</div>
|
|
<div class="views-element-container related-articles block block-views block-views-blockrelated-articles-related-articles">
|
|
|
|
<h2 class="related-articles__title">Related Content</h2>
|
|
|
|
<div><div class="view view-related-articles view-id-related_articles view-display-id-related_articles js-view-dom-id-5cbbc2c972f80442d9d5114ead80fc357cb9bb11599e0c4f497fafe6ab62bf3e">
|
|
|
|
|
|
|
|
<div class="view-content">
|
|
<div class="views-row"><div class="views-field views-field-field-lead-image"><div class="field-content"> <a href="/article/23/2/automate-common-tasks-rundeck" hreflang="en"><img loading="lazy" src="/sites/default/files/styles/222x125/public/lead-images/team_dev_email_chat_video_work_wfm_desk_520.png?itok=H-wvpztI" width="222" height="125" alt="Working on a team, busy worklife" title="Working on a team, busy worklife" class="image-style-_22x125">
|
|
|
|
</a>
|
|
</div></div><div class="views-field views-field-title"><span class="field-content"><a href="/article/23/2/automate-common-tasks-rundeck" hreflang="en">Delegate common tasks with an open source automation tool</a></span></div></div>
|
|
<div class="views-row"><div class="views-field views-field-field-lead-image"><div class="field-content"> <a href="/article/23/2/open-source-mitigate-risk" hreflang="en"><img loading="lazy" src="/sites/default/files/styles/222x125/public/lead-images/BIZ_question_B.png?itok=yy1CXqVy" width="222" height="125" alt="A bunch of question marks" title="A bunch of question marks" class="image-style-_22x125">
|
|
|
|
</a>
|
|
</div></div><div class="views-field views-field-title"><span class="field-content"><a href="/article/23/2/open-source-mitigate-risk" hreflang="en">4 questions open source engineers should ask to mitigate risk at scale</a></span></div></div>
|
|
<div class="views-row"><div class="views-field views-field-field-lead-image"><div class="field-content"> <a href="/article/23/2/gherkin-language-developers" hreflang="en"><img loading="lazy" src="/sites/default/files/styles/222x125/public/lead-images/practicing-empathy.jpg?itok=2mvA2FoR" width="222" height="125" alt="Practicing empathy" title="Practicing empathy" class="image-style-_22x125">
|
|
|
|
</a>
|
|
</div></div><div class="views-field views-field-title"><span class="field-content"><a href="/article/23/2/gherkin-language-developers" hreflang="en">How the Gherkin language bridges the gap between customers and developers</a></span></div></div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-default-license">
|
|
|
|
|
|
|
|
<div class="field field--name-field-default-license field--type-list-string field--label-hidden field__item"><a rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/">
|
|
<img alt="Creative Commons License" src="/themes/osdc/assets/img/cc-by-sa-4.png" title="This work is licensed under a Creative Commons Attribution-Share Alike 4.0 International License."></a>This work is licensed under a Creative Commons Attribution-Share Alike 4.0 International License.</div>
|
|
|
|
</div>
|
|
<div class="green-on-gray cta-block block block-layout-builder block-field-blocknodearticlefield-cta-ref">
|
|
|
|
|
|
|
|
<div class="field field--name-field-cta-ref field--type-entity-reference field--label-hidden field__item">
|
|
<div class="clearfix text-formatted field field--name-field-cta-content field--type-text-long field--label-hidden field__item"><h2>Download the ultimate DevOps hiring guide</h2>
|
|
<p>Build your DevOps team with these best practices for prospective employees and hiring managers.</p>
|
|
</div>
|
|
|
|
<div class="field field--name-field-cta-link field--type-link field--label-hidden field__item"><a href="/downloads/devops-hiring-guide">Download now</a></div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
</article>
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</main>
|
|
|
|
<footer class="global_footer">
|
|
<div class="wrapper">
|
|
<div class="region region-left-footer">
|
|
<div id="block-aboutthissitefooter" class="block block-block-content block-block-content1d00183d-1f84-4ed3-9367-c632014cd322">
|
|
|
|
<h2>About This Site</h2>
|
|
|
|
|
|
<div class="clearfix text-formatted field field--name-body field--type-text-with-summary field--label-hidden field__item"><p>The opinions expressed on this website are those of each author, not of the author's employer or of Red Hat.</p>
|
|
<p><a href="/">Opensource.com</a> aspires to publish all content under a <a href="https://creativecommons.org/licenses/">Creative Commons license</a> but may not be able to do so in all cases. You are responsible for ensuring that you have the necessary permission to reuse any work on this site. Red Hat and the Red Hat logo are trademarks of Red Hat, LLC, registered in the United States and other countries.</p>
|
|
<p>A note on advertising: Opensource.com does not sell advertising on the site or in any of its newsletters.</p>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="region region-right-footer">
|
|
<div id="block-sitebranding" class="block block-system block-system-branding-block">
|
|
|
|
|
|
<div class="branding">
|
|
<a href="/" rel="home" aria-label class="branding__logo">
|
|
<img src="/themes/osdc/logo.svg" alt="Home">
|
|
</a>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="region region-copyright">
|
|
<div id="block-copyrightfooter" class="block block-block-content block-block-contentc0a744a4-2740-4f8a-a693-ee8d7cb5ff86">
|
|
|
|
|
|
|
|
<div class="clearfix text-formatted field field--name-body field--type-text-with-summary field--label-hidden field__item"><p>Copyright ©<span id="copyright-year">2021</span> Red Hat, LLC</p>
|
|
</div>
|
|
|
|
</div>
|
|
<nav role="navigation" aria-labelledby="block-legal-menu" id="block-legal" class="block block-menu navigation menu--legal">
|
|
|
|
<h2 class="visually-hidden" id="block-legal-menu">Legal</h2>
|
|
|
|
|
|
|
|
<ul class="menu">
|
|
<li class="menu-item">
|
|
<a href="/privacy-policy" data-drupal-link-system-path="node/17705">Privacy Policy</a>
|
|
</li>
|
|
<li class="menu-item">
|
|
<a href="/legal" data-drupal-link-system-path="legal">Terms of use</a>
|
|
</li>
|
|
<li class="menu-item">
|
|
<span id="teconsent"></span>
|
|
</li>
|
|
</ul>
|
|
|
|
|
|
|
|
</nav>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
<div id="consent_blackbar"></div>
|
|
|
|
</footer>
|
|
</div>
|
|
</div>
|
|
|
|
<cpx-reporter event="User Detected" auto><script type="data">{"loggedIn":false}</script>
|
|
</cpx-reporter>
|
|
<cpx-reporter auto event="Page Load Completed"></cpx-reporter>
|
|
<script type="application/json" data-drupal-selector="drupal-settings-json">{"path":{"baseUrl":"\/","pathPrefix":"","currentPath":"node\/47266","currentPathIsAdmin":false,"isFront":false,"currentLanguage":"en"},"pluralDelimiter":"\u0003","suppressDeprecationErrors":true,"ajaxPageState":{"libraries":"eJxVUdt2xCAI_CGNn5RjlE3cJWIBs02_vuTS0_ZBzjBcBxNGkT1MUcClC68gEmeQH79StpgZqaU10PAHD0uZF7SnT8snhlCqAteIw_OjA-_Dg3i9ItVQxPIFLkeNGHfgcQU1PIfUPkeGRmzFY6K1UYWqzpKo65iLJNqsWzA6ETqSnEJkLQnBK9jyfHNdF2I_IaXXP-YYdBFJ4xX3DI-LmpGmeHdthDdiwKiQfSKTZMucpEDktLgjK2TuzYQe2GOpL3FsBaHXjRT8AthsrZN6e136OklvjheTl2Ebox1pNwUSsnX55Y_jjOd1wj3ZF4XVJyzpBdnJLuZeP7YVeEs47bBS7gjf88Wz6A","theme":"osdc","theme_token":null},"ajaxTrustedUrl":{"form_action_p_pvdeGsVG5zNF_XLGPTvYSKCf43t8qZYSwcfZl2uzM":true,"\/article\/18\/8\/now-available-open-source-guide-devops-monitoring-tools?ajax_form=1":true,"\/site-search":true},"ajax":{"edit-submit--2":{"disable-refocus":true,"callback":"::ajaxSubmit","event":"click","wrapper":"rate-widget-base-form","progress":{"type":null},"url":"\/article\/18\/8\/now-available-open-source-guide-devops-monitoring-tools?ajax_form=1","httpMethod":"POST","dialogType":"ajax","submit":{"_triggering_element_name":"op","_triggering_element_value":"Save"}}},"user":{"uid":0,"permissionsHash":"bc13164d8931f8d14b743e3c22c9159308b51a70c1eafe240ce98221f01ff908"}}</script>
|
|
<script src="/sites/default/files/js/js_Wlv6wa8kL6PCkC1orXiTY8r2QgMkdPttdi_yc1ADois.js?scope=footer&delta=0&language=en&theme=osdc&include=eJxNjmtuxCAMhC8UwpEiL0wDW4OpMdvm9vX2ofaHpU8zHo8zGTFd0KPBnM-YxsehGKLmYpI2pKPbpsU543Fk3zq-IjFJN_dCNbSQuKZX5E1mTvFkuRF_My0rouF5_0dQq4kRDDShm5Ihrv4QQyjg8Su9Byur3eYaWxJFrF6mnXi_vy3otb-INncyZq9jwOI_3ks9C_vYff69Tp6-vHvGPIg_AXjUYko"></script>
|
|
<script src="/themes/osdc/node_modules/%40patternfly/pfe-readtime/dist/pfe-readtime.min.js?sqka6y" type="module"></script>
|
|
<script src="/themes/osdc/node_modules/%40patternfly/pfe-avatar/dist/pfe-avatar.min.js?sqka6y" type="module"></script>
|
|
<script src="/sites/default/files/js/js_6SxNdRkRYvBIPr9ggCW9ZOVwSz2Kzy0raMKUAY_0mhU.js?scope=footer&delta=3&language=en&theme=osdc&include=eJxNjmtuxCAMhC8UwpEiL0wDW4OpMdvm9vX2ofaHpU8zHo8zGTFd0KPBnM-YxsehGKLmYpI2pKPbpsU543Fk3zq-IjFJN_dCNbSQuKZX5E1mTvFkuRF_My0rouF5_0dQq4kRDDShm5Ihrv4QQyjg8Su9Byur3eYaWxJFrF6mnXi_vy3otb-INncyZq9jwOI_3ks9C_vYff69Tp6-vHvGPIg_AXjUYko"></script>
|
|
|
|
</body>
|
|
</html>
|