1910 lines
58 KiB
HTML
1910 lines
58 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en-us">
|
||
|
||
<head>
|
||
<link href="https://gmpg.org/xfn/11" rel="profile" />
|
||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
|
||
|
||
<!-- Enable responsiveness on mobile devices-->
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1" />
|
||
|
||
<title>
|
||
|
||
Targeted Diagnostic Logging in Production · Terse Systems
|
||
|
||
</title>
|
||
|
||
<!-- Twitter Card -->
|
||
<meta name="twitter:card" content="summary" />
|
||
|
||
<meta name="twitter:title" content="Targeted Diagnostic Logging in Production" />
|
||
<meta name="twitter:description" content="" />
|
||
<meta name="twitter:url" content="https://tersesystems.com/blog/2019/07/22/targeted-diagnostic-logging-in-production/" />
|
||
|
||
|
||
|
||
|
||
|
||
<!-- CSS -->
|
||
<link rel="stylesheet" href="/assets/css/main.css" />
|
||
|
||
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Abril+Fatface" />
|
||
|
||
<!-- Fonts -->
|
||
|
||
<link href='//fonts.googleapis.com/css?family=Merriweather:900,900italic,300,300italic' rel='stylesheet' type='text/css'>
|
||
<link href='//fonts.googleapis.com/css?family=Lato:900,300' rel='stylesheet' type='text/css'>
|
||
|
||
|
||
<link href="//maxcdn.bootstrapcdn.com/font-awesome/latest/css/font-awesome.min.css" rel="stylesheet">
|
||
|
||
|
||
|
||
<!-- Icons -->
|
||
<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/favicon.png" />
|
||
<link rel="shortcut icon" href="/favicon.ico" />
|
||
|
||
<!-- RSS -->
|
||
<link rel="alternate" type="application/rss+xml" title="RSS" href="/feed.xml" />
|
||
|
||
<!-- Additional head bits without overriding original head -->
|
||
<link rel="dns-prefetch" href="//maxcdn.bootstrapcdn.com">
|
||
<link rel="dns-prefetch" href="//cdnjs.cloudflare.com">
|
||
|
||
<script src="https://cdnjs.cloudflare.com/ajax/libs/mermaid/10.6.1/mermaid.min.js"></script>
|
||
<script>
|
||
document.addEventListener('DOMContentLoaded', function() {
|
||
// Initialize mermaid with default config
|
||
mermaid.initialize({
|
||
startOnLoad: false,
|
||
theme: 'default'
|
||
});
|
||
|
||
// Find all pre elements containing code with language-mermaid class
|
||
document.querySelectorAll('pre code.language-mermaid').forEach(function(element) {
|
||
// Create a div to hold the diagram
|
||
var div = document.createElement('div');
|
||
div.classList.add('mermaid');
|
||
div.textContent = element.textContent;
|
||
|
||
// Replace the pre element with our new div
|
||
element.parentElement.replaceWith(div);
|
||
});
|
||
|
||
// Run mermaid on the newly created divs
|
||
mermaid.run();
|
||
});
|
||
</script>
|
||
|
||
</head>
|
||
|
||
|
||
<body class="post">
|
||
|
||
<div id="sidebar">
|
||
<header>
|
||
<div class="site-title">
|
||
<a href="/">
|
||
|
||
<span class="back-arrow icon"><svg fill="#000000" height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M0 0h24v24H0z" fill="none"/>
|
||
<path d="M20 11H7.83l5.59-5.59L12 4l-8 8 8 8 1.41-1.41L7.83 13H20v-2z"/>
|
||
</svg></span>
|
||
|
||
Terse Systems
|
||
</a>
|
||
</div>
|
||
<p class="lead"></p>
|
||
</header>
|
||
|
||
<nav id="sidebar-nav-links">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div id="category-links">
|
||
<h3 class="site-heading">categories</h3>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div><a class="category-link "
|
||
href="/category/industry/">Industry</a></div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div><a class="category-link "
|
||
href="/category/life/">Life</a></div>
|
||
|
||
|
||
|
||
|
||
|
||
<div><a class="category-link "
|
||
href="/category/llm/">LLM</a></div>
|
||
|
||
|
||
|
||
|
||
|
||
<div><a class="category-link "
|
||
href="/category/logging/">Logging</a></div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div><a class="category-link "
|
||
href="/category/security/">Security</a></div>
|
||
|
||
|
||
|
||
|
||
|
||
<div><a class="category-link "
|
||
href="/category/software/">Software</a></div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<!-- Optional additional links to insert in sidebar nav -->
|
||
<div class="social-icons">
|
||
<div class="social-icons-right">
|
||
|
||
<a class="fa fa-github" href="https://github.com/wsargent"></a>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<a class="fa fa-rss" href="/feed.xml"></a>
|
||
<a class="fa fa-envelope" href="/cdn-cgi/l/email-protection#0f786663634f7b6a7d7c6a7c767c7b6a627c216c6062"></a>
|
||
</div>
|
||
<div class="right">
|
||
|
||
|
||
|
||
</div>
|
||
</div>
|
||
<div class="clearfix"></div>
|
||
|
||
</nav>
|
||
|
||
</div>
|
||
|
||
<main class="container">
|
||
<header>
|
||
<h1 class="post-title">Targeted Diagnostic Logging in Production</h1>
|
||
</header>
|
||
<div class="content">
|
||
<div class="post-meta">
|
||
<span class="post-date">22 Jul 2019</span>
|
||
<span class="post-categories">
|
||
|
||
•
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<a href="/category/logging/">
|
||
logging
|
||
</a>
|
||
|
||
|
||
</span>
|
||
</div>
|
||
|
||
<div class="post-body">
|
||
<p>TL;DR Diagnostic logging is typically not available in production, because of concerns that logging information at DEBUG level is indiscriminate. This blog post shows how to combine diagnostic logging with feature flag management to provide targeted debug information in production only for specific groups, users, or sessions.</p>
|
||
|
||
<h2 id="the-definition">The Definition</h2>
|
||
|
||
<p>Diagnostic logging is defined here as "debug logging statements with an audience." In constrast to debug logging statements that are used for internal debugging and are for the developer alone, diagnostic logging is written and maintained for the development team, or for operations, or both.</p>
|
||
|
||
<p>A diagnostic logging statement should be as useful in production as code that records a metric, or sets up a distributed tracing span. These systems produce operational data that make systems observable. If business logic and data is blood, then observability is lymph. Imagine <a href="https://en.wikipedia.org/wiki/Circulatory_system">circulatory system</a> and the <a href="https://en.wikipedia.org/wiki/Lymphatic_system">lymphatic system</a>, running in parallel to each other. Lymph is hidden and never seen, but it monitors and protects at all times. Likewise, diagnostic logging is usually not visible until there's a problem.</p>
|
||
|
||
<p>UPDATE: I have a new blog post with a <a href="https://tersesystems.com/blog/2019/10/05/diagnostic-logging-citations-and-sources/">full definition of diagnostic logging wherein I cite my sources</a>.</p>
|
||
|
||
<h2 id="the-problem">The Problem</h2>
|
||
|
||
<p>You're running a complex stateful web application, or a set of them, all talking to each other using gRPC, Kafka, Akka Artery, Aeron, etc. Real cattle-not-sheep-k8s-in-the-cloud type stuff. Something is going wrong in production, and we don't know why. Some data comes into the system, is processed, and produces the wrong result, like a negative price. But it only happens in Europe, and only for users who use Firefox.</p>
|
||
|
||
<p>There are a specific set of tools that are available in production, but none of them will help in this circumstance. Dropwizard Metrics will tell you how many times it happened, but it won't tell you why it happened. The Opentracing API will tell you the trace and the spans and you can log information in a particular span, but it works on a sampling basis and this happens rarely enough that you can't catch it. Using an SLF4J logging framework at the standard INFO level won't help: there's no INFO level logs telling you what happened. The exception that is thrown happens long after the calculation has taken place, and can't tell you the internal state and flow leading up to that point.</p>
|
||
|
||
<p>You can argue that testing should have caught it. But tests have problems <a href="https://stackoverflow.com/questions/2029509/how-to-model-concurrency-in-unit-tests">accounting for concurrency</a>, and distributed architectures are complex concurrent systems by design. Tests can only deal with the code paths and the interactions that you as a programmer have considered and accounted for, which is a problem when you have multiple systems interacting a concurrent manner. Most problems in distributed systems are ones <a href="https://thenewstack.io/honeycombs-charity-majors-go-ahead-test-in-production/">involving high-cardinality</a>: it's <a href="https://www.slideshare.net/bcantrill/zebras-all-the-way-down-the-engineering-challenges-of-the-data-path">Zebras All The Way Down</a>. Tests can catch many use cases, but you will also be <a href="https://twitter.com/mipsytipsy/status/1149608597078196224">testing in production</a>.</p>
|
||
|
||
<h2 id="the-dichotomy">The Dichotomy</h2>
|
||
|
||
<p>If this were happening in development, this would be easy. Start up the server on your local machine, remote attach through IntelliJ IDEA's debugger, and set a breakpoint where you think things are going wrong.</p>
|
||
|
||
<p>Or maybe it's a bit more complex than that, and you've got several services you need to run through, or want to compare logs. You'd add a bunch of debugging statements, turn the logger level up to DEBUG, and poke around.</p>
|
||
|
||
<p>The point here is that debugging involves exposing some internal state, which increases observability.</p>
|
||
|
||
<p>Using <a href="https://twitter.com/mipsytipsy/status/1151972148224729088">Charity Majors</a>' definition:</p>
|
||
|
||
<blockquote>
|
||
<p>Observability is how well you can understand the inner workings of your system simply by observing its outputs. Look it up. This MATTERS, because observability is how you debug and understand unknown-unknown states, ones you've never seen before and could not have predicted.</p>
|
||
</blockquote>
|
||
|
||
<p>Following on from this, <a href="https://twitter.com/mipsytipsy/status/1150951315624742912">the goal of observability</a>:</p>
|
||
|
||
<blockquote>
|
||
<p>The holy grail of observability is the ability to be able to ask any question, understand any previously unseen state your system may get itself into; without having to ship new code to handle that state (bc that implies you knew enough to predict it)</p>
|
||
</blockquote>
|
||
|
||
<p>Diagnostic logging is a safe, performant, domain aware way of making internal state observable. But diagnostic logging is typically not available in production.</p>
|
||
|
||
<p>Why is that?</p>
|
||
|
||
<h2 id="indiscriminate-diagnostic-logging">Indiscriminate Diagnostic Logging</h2>
|
||
|
||
<p>Diagnostic logging is not used in production because the operational tools available to manage logging are crude and indiscriminate.</p>
|
||
|
||
<p>In many cases, logging levels cannot be changed in a running system. To change the logging levels, the service must be restarted with new configuration. In most systems, there is no facility to change logging levels on the fly. Even systems that do have this capability like <a href="https://twitter.github.io/twitter-server/Features.html#dynamically-change-log-levels">Twitter Server</a> will only change the logger level by an individual server, which is little help when dealing with distributed systems.</p>
|
||
|
||
<p>Even in cases where logging levels can be changed on the fly, logging levels are directly tied to the logger – in SLF4J, this is typically the class name. As such, turning the logger to DEBUG level means that all interaction with any instances of that class are logged at debug level, typically at high volume.</p>
|
||
|
||
<p>When there is a high volume of logs, it does two things. First, it exhausts the logging budget: both the cost of storage and the cost of indexing go up dramatically. Second, it has the risk of causing non-linear performance problems in the server, as the overhead of logging at high rates can stress the IO and network capacity of instances.</p>
|
||
|
||
<p>As Cindy Sridharan says in <a href="https://medium.com/@copyconstruct/logs-and-metrics-6d34d3026e38">Logs and Metrics</a> says:</p>
|
||
|
||
<blockquote>
|
||
<p>By far, the biggest drawbacks of logs is how operationally and monetarily expensive they are to process and store. Albeit structured logging doesn’t suffer from some of the downsides inherent in pure string-based log manipulation, it’s still far more voluminous to store as well as slower to query and process, as evident from the pricing models of log (event) based monitoring vendors.</p>
|
||
</blockquote>
|
||
|
||
<p>Because indiscriminate logging has both high known costs and unknown risks, and logging frameworks provide few out of the box mitigations such as <a href="https://tersesystems.com/blog/2019/06/15/application-logging-in-java-part-9/">budget aware logging</a>, sometimes operations will simply not allow diagnostic logging in production, because they don't know how to do it safely.</p>
|
||
|
||
<h2 id="effective-diagnostic-logging">Effective Diagnostic Logging</h2>
|
||
|
||
<p>But diagnostic logging is vital in production. You simply <a href="https://ayende.com/blog/165633/logging-production-systems">must have logs in production</a>. <a href="http://highscalability.com/log-everything-all-time">Log Everything All the Time</a> puts it very succinctly:</p>
|
||
|
||
<blockquote>
|
||
<p>To see why the typical logging approach is broken, imagine this scenario: Your site has been up and running great for weeks. No problems. A foreshadowing beeper goes off at 2AM. It seems some users can no longer add comments to threads. Then you hear the debugging deathknell: it's an intermittent problem and customers are pissed. Fix it. Now.</p>
|
||
|
||
<p>So how are you going to debug this? The monitoring system doesn't show any obvious problems or errors. You quickly post a comment and it works fine. This won't be easy. So you think. Commenting involves a bunch of servers and networks. There's the load balancer, spam filter, web server, database server, caching server, file server, and a few networks switches and routers along the way. Where did the fault happen? What went wrong?</p>
|
||
|
||
<p>All you have at this point are your logs. You can't turn on more logging because the Heisenberg already happened. You can't stop the system because your system must always be up. You can't deploy a new build with more logging because that build has not been tested and you have no idea when the problem will happen again anyway. Attaching a debugger to a process, while heroic sounding, doesn't help at all.</p>
|
||
|
||
<p>What you need to be able to do is trace though all relevant logs, pull together a time line of all relevant operations, and see what happened. And this is where trace/info etc is useless. You don't need function/method traces. You need a log of all the interesting things that happened in the system. Knowing "func1" was called is of no help. You need to know all the parameters that were passed to the function. You need to know the return value from the function. Along with anything else interesting it did.</p>
|
||
</blockquote>
|
||
|
||
<p>A brief overview of Stack Overflow supports that debug statements are, in fact, meant for production scenarios:</p>
|
||
|
||
<p><a href="https://softwareengineering.stackexchange.com/a/279822/20456">Martin Ba</a>:</p>
|
||
|
||
<blockquote>
|
||
<p>DEBUG logging must be able to be used in production (but tends to still be off normally)
|
||
TRACE logging is allowed to be such that using it in production (other than for a specific / short session) is infeasible</p>
|
||
</blockquote>
|
||
|
||
<p><a href="https://softwareengineering.stackexchange.com/a/279694/20456">user22815</a>:</p>
|
||
|
||
<blockquote>
|
||
<p>DEBUG: used for dumping variable state, specific error codes, etc. For example: a web service might return error code 809214, which could be logged while the application tells the user "communication failed." Imagine a developer receiving a log from a user's system long after the error occurred and wondering "why did the failure occur?" that is a good thing to log at the debug level. Another example might be if a bug keeps occurring in production but is hard to reproduce, debug log certain variables or events in the troublesome module to help tell developers the program state when the error occurs to help troubleshooting.</p>
|
||
</blockquote>
|
||
|
||
<p><a href="https://stackoverflow.com/a/2031209/5266">GrayWizardx</a>:</p>
|
||
|
||
<blockquote>
|
||
<p>Debug - Information that is diagnostically helpful to people more than just developers (IT, sysadmins, etc.).</p>
|
||
</blockquote>
|
||
|
||
<p>The usual compromise to indiscriminate debug logging is to have <a href="https://opentracing.io/docs/overview/what-is-tracing/">distributed tracing</a>, but that is subject to <a href="https://www.jaegertracing.io/docs/1.13/sampling/">sampling</a>, can only provide data limited to instrumentation libraries (unless you manually control it yourself), and doesn't take into account the information that we do have: we already know it happens in a particular region, with a particular browser. For effective diagnostic logging issue is that we'd like to be discriminating about what debug statements show up. We want something more powerful than searching through whatever made it through random sampling. Rather than logging by the class name or by log level, we'd like to say "for the specific users we think are affected by this error, log everything that they do at DEBUG level" and still keep everyone else at INFO level. In short, we want <a href="https://en.wikipedia.org/wiki/Targeted_advertising">targeting</a>.</p>
|
||
|
||
<h2 id="targeted-diagnostic-logging-using-logback-and-launchdarkly">Targeted Diagnostic Logging using Logback and LaunchDarkly</h2>
|
||
|
||
<p>Logback has the idea of <a href="https://logback.qos.ch/manual/filters.html#TurboFilter">turbo filters</a>, which are filters that determine whether a logging event should be created or not. They are not appender specific in the way that normal filters are, and so are used to override logger levels on a per call basis. Turbo filters can be combined with marker-based logic that allows the application to decide when a logging statement can be generated.</p>
|
||
|
||
<p>This works especially well with a feature flagging service like <a href="https://docs.launchdarkly.com/docs/java-sdk-reference#section-variation">Launch Darkly</a>, where you can <a href="https://docs.launchdarkly.com/docs/targeting-users#section-assigning-users-to-a-variation">target particular users</a> and set up logging based on the user variation – thanks to <a href="https://felixsargent.com/">Felix Sargent</a> for making the connection here. This gives us <a href="https://launchdarkly.com/blog/launched-custom-targeting-rules/">high cardinality targeting rules</a> for free.</p>
|
||
|
||
<p>Using the debug statement is a matter of adding the user-specific marker, <a href="https://docs.launchdarkly.com/docs/java-sdk-reference#section-users">ensuring that it has the elements</a> you need to filter:</p>
|
||
|
||
<div class="language-java highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="kd">public</span> <span class="kd">class</span> <span class="nc">Main</span> <span class="o">{</span>
|
||
<span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">IOException</span> <span class="o">{</span>
|
||
<span class="n">LDClient</span> <span class="n">client</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LDClient</span><span class="o">(</span><span class="s">"sdk-key"</span><span class="o">);</span>
|
||
<span class="n">LDMarkerFactory</span> <span class="n">markerFactory</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LDMarkerFactory</span><span class="o">(</span><span class="n">client</span><span class="o">);</span>
|
||
<span class="n">LDUser</span> <span class="n">ldUser</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LDUser</span><span class="o">.</span><span class="na">Builder</span><span class="o">(</span><span class="s">"http-session-id"</span><span class="o">)</span>
|
||
<span class="o">.</span><span class="na">country</span><span class="o">(</span><span class="n">country</span><span class="o">)</span>
|
||
<span class="o">.</span><span class="na">customString</span><span class="o">(</span><span class="s">"userAgent"</span><span class="o">,</span> <span class="n">userAgent</span><span class="o">)</span>
|
||
<span class="o">.</span><span class="na">customString</span><span class="o">(</span><span class="s">"browser"</span><span class="o">,</span> <span class="n">uaDetection</span><span class="o">(</span><span class="n">userAgent</span><span class="o">))</span>
|
||
<span class="o">.</span><span class="na">build</span><span class="o">();</span>
|
||
<span class="n">LDMarkerFactory</span><span class="o">.</span><span class="na">LDMarker</span> <span class="n">ldMarker</span> <span class="o">=</span> <span class="n">markerFactory</span><span class="o">.</span><span class="na">create</span><span class="o">(</span><span class="s">"debug.flag"</span><span class="o">,</span> <span class="n">ldUser</span><span class="o">);</span>
|
||
|
||
<span class="n">Logger</span> <span class="n">logger</span> <span class="o">=</span> <span class="n">LoggerFactory</span><span class="o">.</span><span class="na">getLogger</span><span class="o">(</span><span class="k">this</span><span class="o">.</span><span class="na">getClass</span><span class="o">());</span>
|
||
<span class="n">logger</span><span class="o">.</span><span class="na">debug</span><span class="o">(</span><span class="n">ldMarker</span><span class="o">,</span> <span class="s">"Hello world, I debug when the debug flag is enabled for this user"</span><span class="o">);</span>
|
||
|
||
<span class="n">client</span><span class="o">.</span><span class="na">close</span><span class="o">();</span>
|
||
<span class="o">}</span>
|
||
<span class="o">}</span>
|
||
</code></pre></div></div>
|
||
|
||
<p>Now, by setting the <code class="language-plaintext highlighter-rouge">country</code> attribute to countries in Europe and setting the <code class="language-plaintext highlighter-rouge">browser</code> attribute to "Firefox", you can get fine-grained diagnostic information that is targeted only to those users who are experiencing the problem.</p>
|
||
|
||
<p>The implementation is available on github at <a href="https://github.com/tersesystems/terse-logback">https://github.com/tersesystems/terse-logback</a>, under the <code class="language-plaintext highlighter-rouge">turbomarker</code> module.</p>
|
||
|
||
<p>Once you've got that information, you need to process it. The best way to work with logs is to use Honeycomb and leverage the <a href="https://www.honeycomb.io/blog/dynamic-sampling-by-example/5/">dynamic sampling</a> to alway sample debug logs, and then leverage Honeycomb's <a href="https://docs.honeycomb.io/working-with-your-data/queries/#working-with-the-query-builder">Query Builder</a> to see all the logs associated with that group.</p>
|
||
|
||
<h2 id="poor-mans-observability">Poor Man's Observability</h2>
|
||
|
||
<p>Charity Majors defines <a href="https://charity.wtf/2019/02/05/logs-vs-structured-events/">logs vs structured events</a> and makes the point that an event can be described as a single, very big high cardinality log statement. Targeted operational (defined as INFO) logging with high cardinality events are useful for observability, because they can help you nail down where a problem is happening.</p>
|
||
|
||
<p>To create poor man's observability, you create a feature flag which can be on or off at a certain rate, and use in your request filter pipeline. Then, if the flag is active, you start <a href="https://www.honeycomb.io/blog/event-foo-moar-context-better-events/">accumulating context</a> using <a href="https://github.com/logstash/logstash-logback-encoder#event-specific-custom-fields">event specific custom fields</a>. Finally, at the end of the request, you write out the request by <a href="https://www.honeycomb.io/blog/structured-logging-and-your-team/">logging as described by Anton Drukh</a> using operational logging.</p>
|
||
|
||
<div class="language-java highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="kd">public</span> <span class="kd">class</span> <span class="nc">HttpApp</span> <span class="o">{</span>
|
||
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">possiblyLogRequest</span><span class="o">()</span> <span class="o">{</span>
|
||
<span class="n">LogstashMarker</span> <span class="n">aggregateMarker</span> <span class="o">=</span> <span class="n">Markers</span><span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="n">ldMarker</span><span class="o">,</span> <span class="o">...);</span>
|
||
<span class="k">if</span> <span class="o">(</span><span class="n">is2xx</span><span class="o">(</span><span class="n">statusCode</span><span class="o">)</span> <span class="o">&&</span> <span class="n">logger</span><span class="o">.</span><span class="na">isInfoEnabled</span><span class="o">(</span><span class="n">aggregateMarker</span><span class="o">))</span> <span class="o">{</span>
|
||
<span class="n">logger</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="n">aggregateMarker</span><span class="o">,</span> <span class="s">"request"</span><span class="o">,</span> <span class="n">kv</span><span class="o">(</span><span class="s">"statusCode"</span><span class="o">,</span> <span class="n">statusCode</span><span class="o">),</span> <span class="n">kv</span><span class="o">(</span><span class="s">"duration"</span><span class="o">,</span> <span class="n">duration</span><span class="o">));</span>
|
||
<span class="o">}</span> <span class="k">else</span> <span class="k">if</span> <span class="o">(</span><span class="n">is4xx</span><span class="o">(</span><span class="n">statusCode</span><span class="o">)</span> <span class="o">&&</span> <span class="n">logger</span><span class="o">.</span><span class="na">isWarnEnabled</span><span class="o">(</span><span class="n">aggregateMarker</span><span class="o">))</span> <span class="o">{</span>
|
||
<span class="n">logger</span><span class="o">.</span><span class="na">warn</span><span class="o">(</span><span class="n">aggregateMarker</span><span class="o">,</span> <span class="s">"request"</span><span class="o">,</span> <span class="n">kv</span><span class="o">(</span><span class="s">"statusCode"</span><span class="o">,</span> <span class="n">statusCode</span><span class="o">),</span> <span class="n">kv</span><span class="o">(</span><span class="s">"duration"</span><span class="o">,</span> <span class="n">duration</span><span class="o">));</span>
|
||
<span class="o">}</span> <span class="k">else</span> <span class="k">if</span> <span class="o">(</span><span class="n">is5xx</span><span class="o">(</span><span class="n">statusCode</span><span class="o">)</span> <span class="o">&&</span> <span class="n">logger</span><span class="o">.</span><span class="na">isErrorEnabled</span><span class="o">(</span><span class="n">aggregateMarker</span><span class="o">))</span> <span class="o">{</span>
|
||
<span class="n">logger</span><span class="o">.</span><span class="na">error</span><span class="o">(</span><span class="n">aggregateMarker</span><span class="o">,</span> <span class="s">"Reply sent"</span><span class="o">,</span> <span class="n">kv</span><span class="o">(</span><span class="s">"statusCode"</span><span class="o">,</span> <span class="n">statusCode</span><span class="o">),</span> <span class="n">kv</span><span class="o">(</span><span class="s">"duration"</span><span class="o">,</span> <span class="n">duration</span><span class="o">));</span>
|
||
<span class="o">}</span>
|
||
<span class="o">}</span>
|
||
<span class="o">}</span>
|
||
</code></pre></div></div>
|
||
|
||
<p>You can change the sampling rate on the feature flag on the fly, and add special processing if you need if you want to treat some requests differently.</p>
|
||
|
||
<p>After that, if you're not <a href="https://tersesystems.com/blog/2019/08/22/tracing-with-logback-and-honeycomb/">appending directly to Honeycomb</a>, you can send log data (either directly or streaming through a log aggregator) to a <a href="https://tersesystems.com/blog/2019/09/18/logging-structured-data-to-database/">database using JSONB</a> using <a href="https://www.komu.engineer/blogs/timescaledb/timescaledb-for-logs">TimescaleDB</a>. You can expire old data with <code class="language-plaintext highlighter-rouge">drop_chunks</code>, and run <a href="https://docs.timescale.com/latest/tutorials/continuous-aggs-tutorial">continuous aggregate</a> queries against JSONB. If you need to do more complex data processing, you can use <a href="https://tersesystems.com/blog/2019/09/28/applying-data-science-to-logs-for-developer-observability/">Apache Spark</a>.</p>
|
||
|
||
<p>If you're asking yourself if this will scale, then my answer is use Honeycomb first, then have a personalized system you can query.</p>
|
||
|
||
<p>Note that this is operational logging: events at INFO or above level that shows the result from input, for the purposes of observability by operations. Diagnostic logging is different: it is typically verbose, messy, and can contain details that operations absolutely doesn't care about.</p>
|
||
|
||
<h2 id="designing-for-diagnostic-logging">Designing for Diagnostic Logging</h2>
|
||
|
||
<p>Good diagnostic logging must be considered as a system design consideration. It means thinking about writing logs as structured logging, rather than as unstructured messages. It means <a href="https://bravenewgeek.com/the-observability-pipeline/">passing a context object through basically everything</a>. It means thinking about cases where a system had to reconnect, and adding a debug statement just in case of <a href="https://blog.acolyer.org/2017/06/15/gray-failure-the-achilles-heel-of-cloud-scale-systems/">gray failure</a>. It means using diagnostic logging when <a href="https://medium.com/@copyconstruct/testing-in-production-the-safe-way-18ca102d0ef1">testing in production</a>.</p>
|
||
|
||
<p>Diagnostic logging is useful for documentation. Unlike comments, diagnostic logging contains compiled flow using existing data, and is "living" in a way that comments are not, because developers read logs but skip over comments.</p>
|
||
|
||
<p>Diagnostic logging is useful for bug fixing and maintenance, especially when log output is included with a bug. The first thing that a programmer will do when working with an unfamiliar codebase is add in debugging statements to figure out what's going on. By establishing a diagnostic logging framework and decent logs, it's that much easier to pick up the flow of events from the bug and tie it back to the code.</p>
|
||
|
||
<p>Diagnostic logging is useful for integration testing. Using structured logging, the <a href="https://alemil.com/how-logging-can-simplify-complex-problems">entire flow can be verified</a> end to end after the integration test has completed, even if the codebase is not technically a "stream events to kafka and do event sourcing" type of system. No special listeners or mocks need to be injected into the system, as the diagnostic logging is in itself an output.</p>
|
||
|
||
<p>Diagnostic logging does not mean that developers can't write some debug statements specifically to fix an issue, and then remove it. It <strong>does</strong> mean that there can be debug statements which are specifically there for production. It <strong>does</strong> mean that contrary to local debugging, developers are not the only audience for diagnostic logging, and must treat operations as the audience. Diagnostic logging must <a href="https://softwareengineering.stackexchange.com/a/235488">capture important decision points</a> in the flow and <a href="https://softwareengineering.stackexchange.com/a/112450">expose only relevant state</a>, as a part of <a href="https://martinfowler.com/articles/domain-oriented-observability.html">domain-oriented observability</a>.</p>
|
||
|
||
<p>Don't worry about getting everything right at once. As soon as you have diagnostic logging as an option, you'll be able to see where to spend your time and effort improving it.</p>
|
||
|
||
<p>As <a href="https://engblog.nextdoor.com/what-to-log-in-production-environments-cc9ad82bdce9">NextDoor Engineering</a> puts it:</p>
|
||
|
||
<blockquote>
|
||
<p>Being able to control the logging levels on running servers has been very useful at Nextdoor, both because we can turn logging levels up in order to debug hard-to-reproduce issues on production, and turn them down to allow us to focus on just some logs. Also, knowing that you can log at a DEBUG level in production makes DEBUG logs more valuable and encourages engineers to write more DEBUG logging.</p>
|
||
</blockquote>
|
||
|
||
<p>If you prefer to keep diagnostic logging segmented away from raw <code class="language-plaintext highlighter-rouge">logger.debug</code> statements intended for internal debugging, then you may want to consider writing a helper to make it clearer. For example, using LaunchDarkly and logstash-logback-encoder, you can do the following</p>
|
||
|
||
<div class="language-java highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="kd">public</span> <span class="kd">class</span> <span class="nc">DiagnosticLoggingExample</span> <span class="o">{</span>
|
||
|
||
<span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="o">{</span>
|
||
<span class="n">Config</span> <span class="n">config</span> <span class="o">=</span> <span class="n">ConfigFactory</span><span class="o">.</span><span class="na">load</span><span class="o">();</span>
|
||
<span class="n">LDClientInterface</span> <span class="n">client</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LDClient</span><span class="o">(</span><span class="n">config</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="s">"launchdarkly.sdkkey"</span><span class="o">));</span>
|
||
<span class="n">Logger</span> <span class="n">logger</span> <span class="o">=</span> <span class="n">LoggerFactory</span><span class="o">.</span><span class="na">getLogger</span><span class="o">(</span><span class="n">Order</span><span class="o">.</span><span class="na">class</span><span class="o">);</span>
|
||
<span class="n">LDMarkerFactory</span> <span class="n">markerFactory</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LDMarkerFactory</span><span class="o">(</span><span class="n">client</span><span class="o">);</span>
|
||
<span class="n">LDUser</span> <span class="n">ldUser</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LDUser</span><span class="o">.</span><span class="na">Builder</span><span class="o">(</span><span class="s">"abc123"</span><span class="o">).</span><span class="na">customString</span><span class="o">(</span><span class="s">"groups"</span><span class="o">,</span> <span class="n">singletonList</span><span class="o">(</span><span class="s">"diagnostic-group"</span><span class="o">)).</span><span class="na">build</span><span class="o">();</span>
|
||
<span class="n">OrderDiagnosticLogging</span> <span class="n">diagnostics</span> <span class="o">=</span> <span class="k">new</span> <span class="n">OrderDiagnosticLogging</span><span class="o">(</span><span class="n">logger</span><span class="o">,</span> <span class="n">markerFactory</span><span class="o">,</span> <span class="n">ldUser</span><span class="o">);</span>
|
||
<span class="n">Order</span> <span class="n">order</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Order</span><span class="o">(</span><span class="s">"id1337"</span><span class="o">,</span> <span class="n">diagnostics</span><span class="o">);</span>
|
||
<span class="n">order</span><span class="o">.</span><span class="na">addToCart</span><span class="o">(</span><span class="k">new</span> <span class="n">LineItem</span><span class="o">());</span>
|
||
<span class="n">order</span><span class="o">.</span><span class="na">checkout</span><span class="o">();</span>
|
||
<span class="o">}</span>
|
||
|
||
<span class="kd">static</span> <span class="kd">class</span> <span class="nc">Order</span> <span class="o">{</span>
|
||
<span class="kd">private</span> <span class="kd">final</span> <span class="n">OrderDiagnosticLogging</span> <span class="n">diagnostics</span><span class="o">;</span>
|
||
|
||
<span class="nd">@JsonProperty</span><span class="o">(</span><span class="s">"id"</span><span class="o">)</span> <span class="c1">// Make available to logstash-logback-encoder</span>
|
||
<span class="kd">private</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">id</span><span class="o">;</span>
|
||
|
||
<span class="kd">public</span> <span class="nf">Order</span><span class="o">(</span><span class="n">String</span> <span class="n">id</span><span class="o">,</span> <span class="n">OrderDiagnosticLogging</span> <span class="n">diagnostics</span><span class="o">)</span> <span class="o">{</span>
|
||
<span class="k">this</span><span class="o">.</span><span class="na">id</span> <span class="o">=</span> <span class="n">id</span><span class="o">;</span>
|
||
<span class="k">this</span><span class="o">.</span><span class="na">diagnostics</span> <span class="o">=</span> <span class="n">diagnostics</span><span class="o">;</span>
|
||
<span class="o">}</span>
|
||
|
||
<span class="kd">public</span> <span class="n">String</span> <span class="nf">getId</span><span class="o">()</span> <span class="o">{</span>
|
||
<span class="k">return</span> <span class="n">id</span><span class="o">;</span>
|
||
<span class="o">}</span>
|
||
|
||
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">addToCart</span><span class="o">(</span><span class="n">LineItem</span> <span class="n">lineItem</span><span class="o">)</span> <span class="o">{</span>
|
||
<span class="n">diagnostics</span><span class="o">.</span><span class="na">reportAddToCart</span><span class="o">(</span><span class="k">this</span><span class="o">,</span> <span class="n">lineItem</span><span class="o">);</span>
|
||
<span class="c1">// ...normal business logic reporting</span>
|
||
<span class="o">}</span>
|
||
|
||
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">checkout</span><span class="o">()</span> <span class="o">{</span>
|
||
<span class="n">diagnostics</span><span class="o">.</span><span class="na">reportCheckout</span><span class="o">(</span><span class="k">this</span><span class="o">);</span>
|
||
<span class="c1">// ...normal business logic reporting</span>
|
||
<span class="o">}</span>
|
||
|
||
<span class="nd">@Override</span>
|
||
<span class="kd">public</span> <span class="n">String</span> <span class="nf">toString</span><span class="o">()</span> <span class="o">{</span>
|
||
<span class="k">return</span> <span class="n">String</span><span class="o">.</span><span class="na">format</span><span class="o">(</span><span class="s">"Order(id = %s)"</span><span class="o">,</span> <span class="n">id</span><span class="o">);</span>
|
||
<span class="o">}</span>
|
||
<span class="o">}</span>
|
||
|
||
<span class="kd">static</span> <span class="kd">class</span> <span class="nc">OrderDiagnosticLogging</span> <span class="o">{</span>
|
||
<span class="kd">private</span> <span class="kd">final</span> <span class="n">Logger</span> <span class="n">logger</span><span class="o">;</span>
|
||
<span class="kd">private</span> <span class="kd">final</span> <span class="n">LDMarkerFactory</span><span class="o">.</span><span class="na">LDMarker</span> <span class="n">ldMarker</span><span class="o">;</span>
|
||
|
||
<span class="n">OrderDiagnosticLogging</span><span class="o">(</span><span class="n">Logger</span> <span class="n">logger</span><span class="o">,</span> <span class="n">LDMarkerFactory</span> <span class="n">markerFactory</span><span class="o">,</span> <span class="n">LDUser</span> <span class="n">ldUser</span><span class="o">)</span> <span class="o">{</span>
|
||
<span class="k">this</span><span class="o">.</span><span class="na">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">;</span>
|
||
<span class="k">this</span><span class="o">.</span><span class="na">ldMarker</span> <span class="o">=</span> <span class="n">markerFactory</span><span class="o">.</span><span class="na">create</span><span class="o">(</span><span class="s">"diagnostics-order"</span><span class="o">,</span> <span class="n">ldUser</span><span class="o">);</span>
|
||
<span class="o">}</span>
|
||
|
||
<span class="kt">void</span> <span class="nf">reportAddToCart</span><span class="o">(</span><span class="n">Order</span> <span class="n">order</span><span class="o">,</span> <span class="n">LineItem</span> <span class="n">lineItem</span><span class="o">)</span> <span class="o">{</span>
|
||
<span class="n">reportArg</span><span class="o">(</span><span class="s">"addToCart"</span><span class="o">,</span> <span class="n">order</span><span class="o">,</span> <span class="n">kv</span><span class="o">(</span><span class="s">"lineItem"</span><span class="o">,</span> <span class="n">lineItem</span><span class="o">));</span>
|
||
<span class="o">}</span>
|
||
|
||
<span class="kt">void</span> <span class="nf">reportCheckout</span><span class="o">(</span><span class="n">Order</span> <span class="n">order</span><span class="o">)</span> <span class="o">{</span>
|
||
<span class="n">report</span><span class="o">(</span><span class="s">"checkout"</span><span class="o">,</span> <span class="n">order</span><span class="o">);</span>
|
||
<span class="o">}</span>
|
||
|
||
<span class="kd">private</span> <span class="kt">void</span> <span class="nf">reportArg</span><span class="o">(</span><span class="n">String</span> <span class="n">methodName</span><span class="o">,</span> <span class="n">Order</span> <span class="n">order</span><span class="o">,</span> <span class="n">StructuredArgument</span> <span class="n">arg</span><span class="o">)</span> <span class="o">{</span>
|
||
<span class="k">if</span> <span class="o">(</span><span class="n">logger</span><span class="o">.</span><span class="na">isDebugEnabled</span><span class="o">(</span><span class="n">ldMarker</span><span class="o">))</span> <span class="o">{</span>
|
||
<span class="n">logger</span><span class="o">.</span><span class="na">debug</span><span class="o">(</span><span class="n">ldMarker</span><span class="o">,</span> <span class="s">"{}: {}, {}"</span><span class="o">,</span> <span class="n">kv</span><span class="o">(</span><span class="s">"method"</span><span class="o">,</span> <span class="n">methodName</span><span class="o">),</span> <span class="n">kv</span><span class="o">(</span><span class="s">"order"</span><span class="o">,</span> <span class="n">order</span><span class="o">),</span> <span class="n">arg</span><span class="o">);</span>
|
||
<span class="o">}</span>
|
||
<span class="o">}</span>
|
||
|
||
<span class="kd">private</span> <span class="kt">void</span> <span class="nf">report</span><span class="o">(</span><span class="n">String</span> <span class="n">methodName</span><span class="o">,</span> <span class="n">Order</span> <span class="n">order</span><span class="o">)</span> <span class="o">{</span>
|
||
<span class="k">if</span> <span class="o">(</span><span class="n">logger</span><span class="o">.</span><span class="na">isDebugEnabled</span><span class="o">(</span><span class="n">ldMarker</span><span class="o">))</span> <span class="o">{</span>
|
||
<span class="n">logger</span><span class="o">.</span><span class="na">debug</span><span class="o">(</span><span class="n">ldMarker</span><span class="o">,</span> <span class="s">"{}: {}"</span><span class="o">,</span> <span class="n">kv</span><span class="o">(</span><span class="s">"method"</span><span class="o">,</span> <span class="n">methodName</span><span class="o">),</span> <span class="n">kv</span><span class="o">(</span><span class="s">"order"</span><span class="o">,</span> <span class="n">order</span><span class="o">));</span>
|
||
<span class="o">}</span>
|
||
<span class="o">}</span>
|
||
<span class="o">}</span>
|
||
<span class="o">}</span>
|
||
</code></pre></div></div>
|
||
|
||
<p>Gives the following iff the user is a member of <code class="language-plaintext highlighter-rouge">diagnostic-group</code>:</p>
|
||
|
||
<div class="language-json highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="p">{</span><span class="w">
|
||
</span><span class="s2">"@timestamp"</span><span class="p">:</span><span class="w"> </span><span class="s2">"2019-07-21T13:11:15.642-07:00"</span><span class="p">,</span><span class="w">
|
||
</span><span class="s2">"@version"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1"</span><span class="p">,</span><span class="w">
|
||
</span><span class="s2">"message"</span><span class="p">:</span><span class="w"> </span><span class="s2">"method=addToCart: order=Order(id = id1337), lineItem=LineItem()"</span><span class="p">,</span><span class="w">
|
||
</span><span class="s2">"logger_name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"com.tersesystems.logback.turbomarker.DiagnosticLoggingExample$Order"</span><span class="p">,</span><span class="w">
|
||
</span><span class="s2">"thread_name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"main"</span><span class="p">,</span><span class="w">
|
||
</span><span class="s2">"level"</span><span class="p">:</span><span class="w"> </span><span class="s2">"DEBUG"</span><span class="p">,</span><span class="w">
|
||
</span><span class="s2">"level_value"</span><span class="p">:</span><span class="w"> </span><span class="mi">10000</span><span class="p">,</span><span class="w">
|
||
</span><span class="s2">"tags"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="w">
|
||
</span><span class="s2">"diagnostics-order"</span><span class="w">
|
||
</span><span class="p">],</span><span class="w">
|
||
</span><span class="s2">"method"</span><span class="p">:</span><span class="w"> </span><span class="s2">"addToCart"</span><span class="p">,</span><span class="w">
|
||
</span><span class="s2">"order"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
|
||
</span><span class="s2">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"id1337"</span><span class="w">
|
||
</span><span class="p">},</span><span class="w">
|
||
</span><span class="s2">"lineItem"</span><span class="p">:</span><span class="w"> </span><span class="p">{}</span><span class="w">
|
||
</span><span class="p">}</span><span class="w">
|
||
</span></code></pre></div></div>
|
||
|
||
<p>Note that the order's attributes are represented in both structured logging representation (the <code class="language-plaintext highlighter-rouge">id</code> attribute) and the traditional the line oriented <code class="language-plaintext highlighter-rouge">toString</code> representation shown in the message. Also note I'm not showing any "context" here, which would normally be user or request info, as it would just be confusing in this example.</p>
|
||
|
||
<h2 id="conclusion">Conclusion</h2>
|
||
|
||
<p>Diagnostic logging provides extra instrumentation in production to explain unknown events, works as documentation explaining the code flow in the source code, and provides much needed context when fixing bugs. Concerns about volume and utility of logging in production can be addressed by precisely modulating and targeting diagnostic logging using feature flags.</p>
|
||
|
||
<h2 id="post-script">Post Script</h2>
|
||
|
||
<p>After writing this blog post, I found inevitably that putting feature flags together with verbose logging is a known practice, although it looks like it's not widespread.</p>
|
||
|
||
<p>Christian Meléndez mentions "3. Verbose Logging for Troubleshooting" in 2018:</p>
|
||
|
||
<blockquote>
|
||
<p>There have been times when I wish I’d enabled the verbose logging in my code—especially when I receive reports that some users have been experiencing errors in code. The first thing I do is try to reproduce the error. If I’m lucky enough and succeed, the next step is to see logs. The sad part of the story is that, at this step, there’s nothing more than “something unexpected happened here.”</p>
|
||
|
||
<p>It’s understandable if you don’t want to have verbose logging because disks could run out of space. No one wants to be awakened in the middle of the night with this news. But what if you could launch your code with verbose logging turned off and only enabled when you need it? Yes, with feature flags, it’s possible. You could activate all different types of logging that you previously configured but released them deactivated. Everyone will be happy with this approach: you get the information only when you need it, and no one has to worry about being paged about a low space problem or purchasing more space just to avoid it.</p>
|
||
|
||
<p>– <a href="https://rollout.io/blog/top-5-use-cases-feature-flags/">The Top 5 Use Cases for Feature Flags</a></p>
|
||
</blockquote>
|
||
|
||
<p>and Willy Schaub writes about feature flags and verbose logging in Azure DevOps, from 2017:</p>
|
||
|
||
<blockquote>
|
||
<p>You would like to include hidden features in your release and enable them for all users in production. For example, you want to be able to collect verbose logging data for troubleshooting. Using a feature flag, you can enable and disable verbose logging as needed.</p>
|
||
|
||
<p>– <a href="https://docs.microsoft.com/en-us/azure/devops/migrate/phase-features-with-feature-flags?view=azure-devops">Explore how to progressively expose your features in production for some or all users</a></p>
|
||
</blockquote>
|
||
|
||
|
||
|
||
|
||
|
||
<div class="post-tags">
|
||
|
||
|
||
<a href="/tags/#terse-logback">
|
||
|
||
<span class="icon">
|
||
<svg fill="#000000" height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M0 0h24v24H0z" fill="none"/>
|
||
<path d="M17.63 5.84C17.27 5.33 16.67 5 16 5L5 5.01C3.9 5.01 3 5.9 3 7v10c0 1.1.9 1.99 2 1.99L16 19c.67 0 1.27-.33 1.63-.84L22 12l-4.37-6.16z"/>
|
||
</svg>
|
||
</span> <span class="tag-name">terse-logback</span>
|
||
</a>
|
||
|
||
|
||
<a href="/tags/#diagnostic-logging">
|
||
|
||
<span class="icon">
|
||
<svg fill="#000000" height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M0 0h24v24H0z" fill="none"/>
|
||
<path d="M17.63 5.84C17.27 5.33 16.67 5 16 5L5 5.01C3.9 5.01 3 5.9 3 7v10c0 1.1.9 1.99 2 1.99L16 19c.67 0 1.27-.33 1.63-.84L22 12l-4.37-6.16z"/>
|
||
</svg>
|
||
</span> <span class="tag-name">diagnostic logging</span>
|
||
</a>
|
||
|
||
|
||
<a href="/tags/#java">
|
||
|
||
<span class="icon">
|
||
<svg fill="#000000" height="24" viewBox="0 0 24 24" width="24" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M0 0h24v24H0z" fill="none"/>
|
||
<path d="M17.63 5.84C17.27 5.33 16.67 5 16 5L5 5.01C3.9 5.01 3 5.9 3 7v10c0 1.1.9 1.99 2 1.99L16 19c.67 0 1.27-.33 1.63-.84L22 12l-4.37-6.16z"/>
|
||
</svg>
|
||
</span> <span class="tag-name">java</span>
|
||
</a>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<section class="comments">
|
||
<h2>Comments</h2>
|
||
|
||
<div id="disqus_thread">
|
||
<button class="disqus-load" onClick="loadDisqusComments()">
|
||
Load Comments
|
||
</button>
|
||
</div>
|
||
<script data-cfasync="false" src="/cdn-cgi/scripts/5c5dd728/cloudflare-static/email-decode.min.js"></script><script>
|
||
|
||
/**
|
||
* RECOMMENDED CONFIGURATION VARIABLES: EDIT AND UNCOMMENT THE SECTION BELOW
|
||
* TO INSERT DYNAMIC VALUES FROM YOUR PLATFORM OR CMS.
|
||
* LEARN WHY DEFINING THESE VARIABLES IS IMPORTANT:s
|
||
* https://disqus.com/admin/universalcode/#configuration-variables
|
||
*/
|
||
var disqus_config = function () {
|
||
this.page.url = "https://tersesystems.com/blog/2019/07/22/targeted-diagnostic-logging-in-production/";
|
||
this.page.identifier = "" ||
|
||
"https://tersesystems.com/blog/2019/07/22/targeted-diagnostic-logging-in-production/";
|
||
}
|
||
function loadDisqusComments() { // DON'T EDIT BELOW THIS LINE
|
||
var d = document, s = d.createElement('script');
|
||
s.src = 'https://tersesystems.disqus.com/embed.js';
|
||
s.setAttribute('data-timestamp', +new Date());
|
||
(d.head || d.body).appendChild(s);
|
||
}
|
||
</script>
|
||
<noscript>
|
||
Please enable JavaScript to view the
|
||
<a href="https://disqus.com/?ref_noscript">comments powered by Disqus</a>.
|
||
</noscript>
|
||
|
||
|
||
</section>
|
||
|
||
<section class="related">
|
||
<h2>Related Posts</h2>
|
||
<ul class="posts-list">
|
||
|
||
<li>
|
||
<h3>
|
||
<a href="/blog/2025/07/27/published-chef-agent/">
|
||
Published Recipe Agent
|
||
<small>27 Jul 2025</small>
|
||
</a>
|
||
</h3>
|
||
</li>
|
||
|
||
<li>
|
||
<h3>
|
||
<a href="/blog/2025/06/21/useful-llm-agent-tools/">
|
||
Useful LLM Agent Tools
|
||
<small>21 Jun 2025</small>
|
||
</a>
|
||
</h3>
|
||
</li>
|
||
|
||
<li>
|
||
<h3>
|
||
<a href="/blog/2025/04/13/writing-an-llm-that-just-works-for-my-brother/">
|
||
Making An LLM That Just Works For My Brother
|
||
<small>13 Apr 2025</small>
|
||
</a>
|
||
</h3>
|
||
</li>
|
||
|
||
</ul>
|
||
</section>
|
||
|
||
</div>
|
||
|
||
</main>
|
||
|
||
<!-- Optional footer content -->
|
||
|
||
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"fa246e4d524c4e3d9e500ed03d473133","r":1,"spa":2}' crossorigin="anonymous"></script>
|
||
</body>
|
||
</html>
|