573 lines
40 KiB
HTML
573 lines
40 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
|
||
<title>Anomaly Alerting in Prometheus</title>
|
||
<meta charset="utf-8" />
|
||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||
<meta name="HandheldFriendly" content="True" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
|
||
<link rel="preload" as="style" href="/assets/built/screen.css?v=S0DSiw6jVHJaYsG0" />
|
||
<link rel="preload" as="script" href="/assets/built/casper.js?v=YtPhrez3GuGLVDYs" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="/assets/built/screen.css?v=S0DSiw6jVHJaYsG0" />
|
||
|
||
<meta name="description" content="Exploring how to do anomaly alerting on seasonal data in Prometheus.">
|
||
<link rel="canonical" href="https://karlstoney.com/response-time-anomaly-alert/">
|
||
<meta name="referrer" content="no-referrer-when-downgrade">
|
||
|
||
<meta property="og:site_name" content="karlstoney.com">
|
||
<meta property="og:type" content="article">
|
||
<meta property="og:title" content="Anomaly Alerting in Prometheus">
|
||
<meta property="og:description" content="Exploring how to do anomaly alerting on seasonal data in Prometheus.">
|
||
<meta property="og:url" content="https://karlstoney.com/response-time-anomaly-alert/">
|
||
<meta property="article:published_time" content="2023-03-08T14:04:15.000Z">
|
||
<meta property="article:modified_time" content="2024-07-01T08:12:06.000Z">
|
||
<meta property="article:tag" content="Prometheus">
|
||
|
||
<meta name="twitter:card" content="summary">
|
||
<meta name="twitter:title" content="Anomaly Alerting in Prometheus">
|
||
<meta name="twitter:description" content="Exploring how to do anomaly alerting on seasonal data in Prometheus.">
|
||
<meta name="twitter:url" content="https://karlstoney.com/response-time-anomaly-alert/">
|
||
<meta name="twitter:label1" content="Written by">
|
||
<meta name="twitter:data1" content="Karl Stoney">
|
||
<meta name="twitter:label2" content="Filed under">
|
||
<meta name="twitter:data2" content="Prometheus">
|
||
<meta name="twitter:site" content="@karlstoney">
|
||
<meta name="twitter:creator" content="@karlstoney">
|
||
|
||
<script type="application/ld+json">
|
||
{
|
||
"@context": "https://schema.org",
|
||
"@type": "Article",
|
||
"publisher": {
|
||
"@type": "Organization",
|
||
"name": "karlstoney.com",
|
||
"url": "https://karlstoney.com/",
|
||
"logo": {
|
||
"@type": "ImageObject",
|
||
"url": "https://karlstoney.com/favicon.ico",
|
||
"width": 48,
|
||
"height": 48
|
||
}
|
||
},
|
||
"author": {
|
||
"@type": "Person",
|
||
"name": "Karl Stoney",
|
||
"image": {
|
||
"@type": "ImageObject",
|
||
"url": "https://karlstoney.com/content/images/2020/02/0.jpeg",
|
||
"width": 200,
|
||
"height": 200
|
||
},
|
||
"url": "https://karlstoney.com/author/karl/",
|
||
"sameAs": [
|
||
"https://karlstoney.com",
|
||
"https://x.com/karlstoney"
|
||
]
|
||
},
|
||
"headline": "Anomaly Alerting in Prometheus",
|
||
"url": "https://karlstoney.com/response-time-anomaly-alert/",
|
||
"datePublished": "2023-03-08T14:04:15.000Z",
|
||
"dateModified": "2024-07-01T08:12:06.000Z",
|
||
"keywords": "Prometheus",
|
||
"description": "Exploring how to do anomaly alerting on seasonal data in Prometheus.",
|
||
"mainEntityOfPage": "https://karlstoney.com/response-time-anomaly-alert/"
|
||
}
|
||
</script>
|
||
|
||
<meta name="generator" content="Ghost 6.62">
|
||
<link rel="alternate" type="application/rss+xml" title="karlstoney.com" href="https://karlstoney.com/rss/">
|
||
<script defer src="https://cdn.jsdelivr.net/ghost/portal@~2.71/umd/portal.min.js" data-i18n="true" data-ghost="https://karlstoney.com/" data-key="1fcef56573b2a5779958db5e8c" data-api="https://karlstoney.com/ghost/api/content/" data-locale="en" crossorigin="anonymous"></script><style id="gh-members-styles">.gh-post-upgrade-cta-content,
|
||
.gh-post-upgrade-cta {
|
||
display: flex;
|
||
flex-direction: column;
|
||
align-items: center;
|
||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
|
||
text-align: center;
|
||
width: 100%;
|
||
color: #ffffff;
|
||
font-size: 16px;
|
||
}
|
||
|
||
.gh-post-upgrade-cta-content {
|
||
border-radius: 8px;
|
||
padding: 40px 4vw;
|
||
}
|
||
|
||
.gh-post-upgrade-cta h2 {
|
||
color: #ffffff;
|
||
font-size: 28px;
|
||
letter-spacing: -0.2px;
|
||
margin: 0;
|
||
padding: 0;
|
||
}
|
||
|
||
.gh-post-upgrade-cta p {
|
||
margin: 20px 0 0;
|
||
padding: 0;
|
||
}
|
||
|
||
.gh-post-upgrade-cta small {
|
||
font-size: 16px;
|
||
letter-spacing: -0.2px;
|
||
}
|
||
|
||
.gh-post-upgrade-cta a {
|
||
color: #ffffff;
|
||
cursor: pointer;
|
||
font-weight: 500;
|
||
box-shadow: none;
|
||
text-decoration: underline;
|
||
}
|
||
|
||
.gh-post-upgrade-cta a:hover {
|
||
color: #ffffff;
|
||
opacity: 0.8;
|
||
box-shadow: none;
|
||
text-decoration: underline;
|
||
}
|
||
|
||
.gh-post-upgrade-cta a.gh-btn {
|
||
display: block;
|
||
background: #ffffff;
|
||
text-decoration: none;
|
||
margin: 28px 0 0;
|
||
padding: 8px 18px;
|
||
border-radius: 4px;
|
||
font-size: 16px;
|
||
font-weight: 600;
|
||
}
|
||
|
||
.gh-post-upgrade-cta a.gh-btn:hover {
|
||
opacity: 0.92;
|
||
}</style><script async src="https://js.stripe.com/v3/"></script>
|
||
<script defer src="https://cdn.jsdelivr.net/ghost/sodo-search@~1.8/umd/sodo-search.min.js" data-key="1fcef56573b2a5779958db5e8c" data-styles="https://cdn.jsdelivr.net/ghost/sodo-search@~1.8/umd/main.css" data-sodo-search="https://karlstoney.com/" data-locale="en" crossorigin="anonymous"></script>
|
||
|
||
<link href="https://karlstoney.com/webmentions/receive/" rel="webmention">
|
||
<script defer src="/public/cards.min.js?v=ShRHxgy4po8zN-Wf"></script>
|
||
<link rel="stylesheet" type="text/css" href="/public/cards.min.css?v=WwnU9jw5ancNC8Gc">
|
||
<script defer src="/public/comment-counts.min.js?v=oFYkaGLdiMqB8VN9" data-ghost-comments-counts-api="https://karlstoney.com/members/api/comments/counts/"></script>
|
||
<script defer src="/public/member-attribution.min.js?v=AKG4hWena9j3yX3I"></script><style>:root {--ghost-accent-color: #15171A;}</style>
|
||
<meta name="msvalidate.01" content="6D7C1C7A2BF9A2CF366B3881023DC3BE" />
|
||
|
||
<style type='text/css'>
|
||
|
||
.gh-powered-by {
|
||
display: none;
|
||
}
|
||
|
||
|
||
</style>
|
||
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.1/css/brands.min.css" crossorigin="anonymous" referrerpolicy="no-referrer" />
|
||
|
||
<style>
|
||
.gh-head-menu .nav-linkedin a,
|
||
.gh-head-menu .nav-mastodon a,
|
||
.gh-head-menu .nav-twitter a,
|
||
.gh-head-menu .nav-x a,
|
||
.gh-head-menu .nav-slack a,
|
||
.gh-head-menu .nav-github a {
|
||
font-size: 0 !important;
|
||
}
|
||
|
||
.gh-head-menu .nav-linkedin a::before,
|
||
.gh-head-menu .nav-mastodon a::before,
|
||
.gh-head-menu .nav-twitter a::before,
|
||
.gh-head-menu .nav-x a::before,
|
||
.gh-head-menu .nav-slack a::before,
|
||
.gh-head-menu .nav-github a::before {
|
||
font-family: "Font Awesome 6 Brands";
|
||
display: inline-block;
|
||
font-size: 20px;
|
||
font-style: normal;
|
||
font-weight: normal;
|
||
font-variant: normal;
|
||
text-rendering: auto;
|
||
-webkit-font-smoothing: antialiased;
|
||
}
|
||
|
||
.gh-head-menu .nav-linkedin a::before {content: "\f08c"}
|
||
.gh-head-menu .nav-mastodon a::before {content: "\f4f6"}
|
||
.gh-head-menu .nav-twitter a::before {content: "\f099"}
|
||
.gh-head-menu .nav-x a::before {content: "\e61b"}
|
||
.gh-head-menu .nav-slack a::before {content: "\f198"}
|
||
.gh-head-menu .nav-github a::before {content: "\f09b"}
|
||
</style>
|
||
|
||
</head>
|
||
<body class="post-template tag-prometheus is-head-left-logo has-cover">
|
||
<div class="viewport">
|
||
|
||
<header id="gh-head" class="gh-head outer">
|
||
<div class="gh-head-inner inner">
|
||
<div class="gh-head-brand">
|
||
<a class="gh-head-logo no-image" href="https://karlstoney.com">
|
||
karlstoney.com
|
||
</a>
|
||
<button class="gh-search gh-icon-btn" aria-label="Search this site" data-ghost-search><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2" width="20" height="20"><path stroke-linecap="round" stroke-linejoin="round" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z"></path></svg></button>
|
||
<button class="gh-burger" aria-label="Main Menu"></button>
|
||
</div>
|
||
|
||
<nav class="gh-head-menu">
|
||
<ul class="nav">
|
||
<li class="nav-home"><a href="https://karlstoney.com/">Home</a></li>
|
||
<li class="nav-about"><a href="https://karlstoney.com/about/">About</a></li>
|
||
<li class="nav-email"><a href="https://karlstoney.com/contact/">Email</a></li>
|
||
<li class="nav-github"><a href="https://github.com/Stono">Github</a></li>
|
||
<li class="nav-twitter"><a href="https://twitter.com/karlstoney">Twitter</a></li>
|
||
<li class="nav-linkedin"><a href="https://www.linkedin.com/in/karl-stoney-94995548/">LinkedIn</a></li>
|
||
</ul>
|
||
|
||
</nav>
|
||
|
||
<div class="gh-head-actions">
|
||
<button class="gh-search gh-icon-btn" aria-label="Search this site" data-ghost-search><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2" width="20" height="20"><path stroke-linecap="round" stroke-linejoin="round" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z"></path></svg></button>
|
||
<div class="gh-head-members">
|
||
<a class="gh-head-link" href="#/portal/signin" data-portal="signin">Sign in</a>
|
||
<a class="gh-head-button" href="#/portal/signup" data-portal="signup">Subscribe</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</header>
|
||
|
||
<div class="site-content">
|
||
|
||
|
||
|
||
|
||
<main id="site-main" class="site-main">
|
||
<article class="article post tag-prometheus featured no-image ">
|
||
|
||
<header class="article-header gh-canvas">
|
||
|
||
<div class="article-tag post-card-tags">
|
||
<span class="post-card-primary-tag">
|
||
<a href="/tag/prometheus/">Prometheus</a>
|
||
</span>
|
||
<span class="post-card-featured"><svg width="16" height="17" viewBox="0 0 16 17" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M4.49365 4.58752C3.53115 6.03752 2.74365 7.70002 2.74365 9.25002C2.74365 10.6424 3.29678 11.9778 4.28134 12.9623C5.26591 13.9469 6.60127 14.5 7.99365 14.5C9.38604 14.5 10.7214 13.9469 11.706 12.9623C12.6905 11.9778 13.2437 10.6424 13.2437 9.25002C13.2437 6.00002 10.9937 3.50002 9.16865 1.68127L6.99365 6.25002L4.49365 4.58752Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path>
|
||
</svg> Featured</span>
|
||
</div>
|
||
|
||
<h1 class="article-title">Anomaly Alerting in Prometheus</h1>
|
||
|
||
<p class="article-excerpt">Exploring how to do anomaly alerting on seasonal data in Prometheus.</p>
|
||
|
||
<div class="article-byline">
|
||
<section class="article-byline-content">
|
||
|
||
<ul class="author-list instapaper_ignore">
|
||
<li class="author-list-item">
|
||
<a href="/author/karl/" class="author-avatar" aria-label="Read more of Karl Stoney">
|
||
<img class="author-profile-image" src="/content/images/size/w100/2020/02/0.jpeg" alt="Karl Stoney" loading="eager" />
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
<div class="article-byline-meta">
|
||
<h4 class="author-name"><a href="/author/karl/">Karl Stoney</a></h4>
|
||
<div class="byline-meta-content">
|
||
<time class="byline-meta-date" datetime="2023-03-08">08 Mar 2023</time>
|
||
<span class="byline-reading-time"><span class="bull">•</span> 7 min read</span>
|
||
</div>
|
||
</div>
|
||
|
||
</section>
|
||
<a href="#/share" class="gh-button gh-button-share">Share</a>
|
||
</div>
|
||
|
||
|
||
</header>
|
||
|
||
<section class="gh-content gh-canvas">
|
||
<p>At <a href="https://careers.autotrader.co.uk/jobs/?ref=karlstoney.com">Auto Trader</a>, we strive to build alerts that the majority can benefit from out-of-the-box, rather than hard coded alerts for specific scenarios. We want you to be able to deploy a service and just get <em>value</em> from the platform - particularly when it comes to observing and alerting on your <a href="https://sre.google/sre-book/monitoring-distributed-systems/?ref=karlstoney.com">Golden Signals</a>, or <a href="https://karlstoney.com/2018/07/07/managing-your-costs-on-kubernetes/">Managing your Costs</a>.</p><p>We've done a pretty good job of this when thinking about obvious threshold based alerts (for example; <code>CPU usage > CPU request for 15 minutes</code> or more), but we strived to improve alerting on anomalies too, to help us detect things like regressions in performance, without defining explicit thresholds.</p><p>In this post I'll be looking at how to use <a href="https://prometheus.io/?ref=karlstoney.com">Prometheus</a> with <a href="https://istio.io/?ref=karlstoney.com">Istio</a>, to do anomaly detection on the response time of your operational services. The implementation will be generic enough that we can apply it globally to all services running on your mesh. You'll be able to apply the same pattern to pretty much any rate-metric you care about.</p><h3 id="recording-our-baseline">Recording our Baseline</h3><p>We are going to be looking at <code>istio_request_duration_milliseconds_bucket</code>, which is a <a href="https://prometheus.io/docs/practices/histograms/?ref=karlstoney.com">Histogram</a>. It also has a lot of dimensions which makes it a very high cardinality metric to work with. The first thing we want to do is roll it up into a simple time series we can work with, using a Recording Rule.</p><pre><code class="language-yaml">- record: "recorded:istio_request_duration_milliseconds_bucket:p95"
|
||
expr: |
|
||
histogram_quantile(
|
||
0.95,
|
||
sum by (destination_workload, le) (
|
||
irate(istio_request_duration_milliseconds_bucket[1m])
|
||
)
|
||
)</code></pre><p>This will give us a nice line for each <code>destination_workload</code> on our cluster. Now in our case, we have 1 workload per namespace. Your setup may be different, in which case you'll need to add <code>destination_workload_namespace</code> too. Or you could use <code>destination_service</code>, if you prefer to group by the FQDN. That's up to you.</p><p>In this example I've also opt'd to only focus on the <code>P95</code> response time, but again that's another decision you'll have to make on what's right for your system.</p><figure class="kg-card kg-image-card"><img src="https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-04-at-11.42.04.png" class="kg-image" alt="" loading="lazy" width="2000" height="775" srcset="https://karlstoney.com/content/images/size/w600/2023/02/Screenshot-2023-02-04-at-11.42.04.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/Screenshot-2023-02-04-at-11.42.04.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/02/Screenshot-2023-02-04-at-11.42.04.png 1600w, https://karlstoney.com/content/images/size/w2400/2023/02/Screenshot-2023-02-04-at-11.42.04.png 2400w" sizes="(min-width: 720px) 720px"></figure><h3 id="building-a-prediction">Building a Prediction</h3><p>The next thing we need to do when we're trying to detect an anomaly, is have a prediction of where we believe the response time should be. The most basic of predictions would be some time based offset from where you are now, say for example; the same time yesterday (if you have a daily trend), or the same time a week ago (if you have a weekly trend). </p><p>At Auto Trader, our patterns are pretty typical when you look at a daily and weekly sample. So we could start by looking at the same time window in the previous week. However we don't want a single outlier week to skew our prediction, so instead - we'll take an average of the last 3 weeks. Unfortunately this is where you'll need to be a bit patient, you'll need to wait until you've got 3 weeks of data recorded using your rule for <code>recorded:istio_request_duration_milliseconds_bucket:p95</code>. Or, you could have a stab at <a href="https://prometheus.io/docs/prometheus/latest/storage/?ref=karlstoney.com#backfilling-for-recording-rules">backfilling</a>, which ill go into at the end.</p><p>Once you've got that in place though, have a quick look at your data over the past few weeks. I'm targetting a single service here for demo purposes, as it has a known spike at a particular time of day:</p><figure class="kg-card kg-image-card"><img src="https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-23-at-08.23.03.png" class="kg-image" alt="" loading="lazy" width="2000" height="1001" srcset="https://karlstoney.com/content/images/size/w600/2023/02/Screenshot-2023-02-23-at-08.23.03.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/Screenshot-2023-02-23-at-08.23.03.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/02/Screenshot-2023-02-23-at-08.23.03.png 1600w, https://karlstoney.com/content/images/size/w2400/2023/02/Screenshot-2023-02-23-at-08.23.03.png 2400w" sizes="(min-width: 720px) 720px"></figure><p>There are a few other things to consider here. We want a relatively smooth prediction. So we can smooth out the signal using an <code>avg_over_time</code> as a windowing function. In order to ensure our average reprsesnts the correct time window, we need to ensure we offset - half that average. So for example here i've decided to average at <code>5m</code>, so instead of offsetting <code>1w</code>, I do <code>6d23h57m30s</code>, so <code>2.5m</code> either side.</p><pre><code>avg_over_time(
|
||
istio_request_duration_milliseconds_bucket:destination:rate1m[5m] offset 6d23h57m30s
|
||
)</code></pre><figure class="kg-card kg-image-card"><img src="https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-23-at-08.48.06.png" class="kg-image" alt="" loading="lazy" width="2000" height="884" srcset="https://karlstoney.com/content/images/size/w600/2023/02/Screenshot-2023-02-23-at-08.48.06.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/Screenshot-2023-02-23-at-08.48.06.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/02/Screenshot-2023-02-23-at-08.48.06.png 1600w, https://karlstoney.com/content/images/size/w2400/2023/02/Screenshot-2023-02-23-at-08.48.06.png 2400w" sizes="(min-width: 720px) 720px"></figure><p>But we should also consider growth/decline when we're looking back. Remember we're trying to create a prediction of where things should be <em>now</em>, not weeks ago. So we can do that by calculating the growth (the average for this week, minus the average for the week we're sampling):</p><pre><code>avg_over_time(
|
||
istio_request_duration_milliseconds_bucket:destination:rate1m[5m] offset 6d23h57m30s
|
||
)
|
||
+
|
||
(
|
||
avg_over_time(
|
||
istio_request_duration_milliseconds_bucket:destination:rate1m[5m]
|
||
)
|
||
-
|
||
avg_over_time(
|
||
istio_request_duration_milliseconds_bucket:destination:rate1m[5m] offset 1w
|
||
)
|
||
)</code></pre><p>Great. So if you apply the same pattern to each of our look back windows, you'll end up with a recording rule that looks something like this. Notice we use the <code>quantile(0.5</code> function to find the median across our weekly samples.</p><pre><code class="language-yaml">- record: anomaly:istio_request_duration_milliseconds_bucket:p95:prediction
|
||
expr: |
|
||
quantile(0.5,
|
||
label_replace(
|
||
(
|
||
avg_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[5m] offset 6d23h57m30s)
|
||
+
|
||
(avg_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[1w]) - avg_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[1w] offset 1w))
|
||
), "offset", "1w", "", ""
|
||
)
|
||
or
|
||
label_replace(
|
||
(
|
||
avg_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[5m] offset 13d23h57m30s)
|
||
+
|
||
(avg_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[1w]) - avg_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[1w] offset 1w))
|
||
), "offset", "2w", "", ""
|
||
)
|
||
or
|
||
label_replace(
|
||
(
|
||
avg_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[5m] offset 20d23h57m30s)
|
||
+
|
||
(avg_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[1w]) - avg_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[1w] offset 1w))
|
||
), "offset", "3w", "", ""
|
||
)
|
||
) without(offset)</code></pre><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-23-at-09.01.04.png" class="kg-image" alt="" loading="lazy" width="2000" height="807" srcset="https://karlstoney.com/content/images/size/w600/2023/02/Screenshot-2023-02-23-at-09.01.04.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/Screenshot-2023-02-23-at-09.01.04.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/02/Screenshot-2023-02-23-at-09.01.04.png 1600w, https://karlstoney.com/content/images/size/w2400/2023/02/Screenshot-2023-02-23-at-09.01.04.png 2400w" sizes="(min-width: 720px) 720px"><figcaption><span style="white-space: pre-wrap;">a lovely smoothed prediction based on the seasonality of our service</span></figcaption></figure><h3 id="calculating-the-z-score">Calculating the Z-Score</h3><p>So we have our prediction, the next thing we want do is create a signal line we can alert on. A z-score (also called a standard score) gives you an idea of how far from the mean (in standard deviations) a data point is. Typically anything more than say, 3 standard deviations from the mean could be considered an anomaly.</p><p>The formula to calculate this is pretty simple: <code>(current value - predicted value) / standard deviation of the value from the past week)</code>. Again, we'll be using another recording rule in order to capture this:</p><pre><code class="language-yaml">- record: anomaly:istio_request_duration_milliseconds_bucket:p95:zscore
|
||
expr: |
|
||
(
|
||
(
|
||
avg_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[5m])
|
||
-
|
||
avg_over_time(anomaly:istio_request_duration_milliseconds_bucket:p95:prediction[5m])
|
||
)
|
||
/ stddev_over_time(recorded:istio_request_duration_milliseconds_bucket:p95[1w])
|
||
)</code></pre><p>So lets take a look at what that looks like across all of our services:</p><figure class="kg-card kg-image-card"><img src="https://karlstoney.com/content/images/2023/02/zscore.png" class="kg-image" alt="" loading="lazy" width="2000" height="782" srcset="https://karlstoney.com/content/images/size/w600/2023/02/zscore.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/zscore.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/02/zscore.png 1600w, https://karlstoney.com/content/images/size/w2400/2023/02/zscore.png 2400w" sizes="(min-width: 720px) 720px"></figure><p>As you can see, the vast majority are sat +/- 1 standard deviations from the prediction, but there are certainly some outliers. Let's take a look at what they highlight.</p><p><strong>Outlier #1 </strong></p><figure class="kg-card kg-image-card"><img src="https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-04-at-11.19.07.png" class="kg-image" alt="" loading="lazy" width="2000" height="780" srcset="https://karlstoney.com/content/images/size/w600/2023/02/Screenshot-2023-02-04-at-11.19.07.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/Screenshot-2023-02-04-at-11.19.07.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/02/Screenshot-2023-02-04-at-11.19.07.png 1600w, https://karlstoney.com/content/images/size/w2400/2023/02/Screenshot-2023-02-04-at-11.19.07.png 2400w" sizes="(min-width: 720px) 720px"></figure><p>Looks like a valid outlier; we certainly had a latency spike.</p><p><strong>Outlier #2 </strong></p><figure class="kg-card kg-image-card"><img src="https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-04-at-11.18.56.png" class="kg-image" alt="" loading="lazy" width="2000" height="769" srcset="https://karlstoney.com/content/images/size/w600/2023/02/Screenshot-2023-02-04-at-11.18.56.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/Screenshot-2023-02-04-at-11.18.56.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/02/Screenshot-2023-02-04-at-11.18.56.png 1600w, https://karlstoney.com/content/images/size/w2400/2023/02/Screenshot-2023-02-04-at-11.18.56.png 2400w" sizes="(min-width: 720px) 720px"></figure><p>Also looks pretty legit. Certainly had a spike.</p><p><strong>Outlier #3 </strong></p><p>This one is certainly more interesting as it's recording a z-score pretty consistently of around -3:</p><figure class="kg-card kg-image-card"><img src="https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-04-at-11.20.57.png" class="kg-image" alt="" loading="lazy" width="2000" height="1020" srcset="https://karlstoney.com/content/images/size/w600/2023/02/Screenshot-2023-02-04-at-11.20.57.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/Screenshot-2023-02-04-at-11.20.57.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/02/Screenshot-2023-02-04-at-11.20.57.png 1600w, https://karlstoney.com/content/images/size/w2400/2023/02/Screenshot-2023-02-04-at-11.20.57.png 2400w" sizes="(min-width: 720px) 720px"></figure><p>If we look at the current value vs the previous week, we can see why. The previous week (and the weeks before it) had consistently higher response times. The Z-score of -3 (ish) is correct in that the current response time is notably lower than where we'd expect it to be, based on history. Looks like someone made some performance improvements! Good effort!</p><h3 id="alerting">Alerting </h3><p>You'll want to gather it for longer periods of time and tweak some of the windowing functions in the examples above to find a threshold that is right for your traffic patterns. The data looks pretty decent for me, so the next thing we want to do is alert on it.</p><p>I'm not that bothered when people make performance improvements, I just want to alert when the response time worsens. We could define that as:</p><blockquote>When the current response time is more than 3 standard deviations from the mean</blockquote><p>Therefore in our case, that's a <code>z-score</code> on our recorded metric of +3.</p><p>We don't want it to be too flakey, it's the internet after all, latency spikes happen. So we'll just alert when the <code>z-score</code> has be +3 for 5 minutes or more.</p><pre><code class="language-yaml">- alert: AnomalyResponseTime
|
||
expr: |
|
||
anomaly:istio_request_duration_milliseconds_bucket:p95:zscore > 3
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
source: '{{ "{{" }}$labels.destination_workload{{ "}}" }}'
|
||
annotations:
|
||
description: >-
|
||
We have detected a potential anomoly in the response times for this service.
|
||
The response time has been > 3 standard deviations from the prediction for more than 5 minutes.
|
||
summary: Anomaly - Response Time</code></pre><p>Of all our data there was only one point it went above 3, and it was for less than 5 minutes so we wouldn't have alerted:</p><figure class="kg-card kg-image-card"><img src="https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-04-at-11.30.45.png" class="kg-image" alt="" loading="lazy" width="2000" height="776" srcset="https://karlstoney.com/content/images/size/w600/2023/02/Screenshot-2023-02-04-at-11.30.45.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/Screenshot-2023-02-04-at-11.30.45.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/02/Screenshot-2023-02-04-at-11.30.45.png 1600w, https://karlstoney.com/content/images/size/w2400/2023/02/Screenshot-2023-02-04-at-11.30.45.png 2400w" sizes="(min-width: 720px) 720px"></figure><p>But here's what it looks like when we do:</p><figure class="kg-card kg-image-card"><img src="https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-04-at-11.31.45.png" class="kg-image" alt="" loading="lazy" width="1172" height="554" srcset="https://karlstoney.com/content/images/size/w600/2023/02/Screenshot-2023-02-04-at-11.31.45.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/Screenshot-2023-02-04-at-11.31.45.png 1000w, https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-04-at-11.31.45.png 1172w" sizes="(min-width: 720px) 720px"></figure><h3 id="backfilling">Backfilling</h3><p><code>promtool</code> has a Backfill capability that allows you to historically backfill data from recording rules. Be warned - it's expensive to run and can take absolutely ages depending on the amount of data you are processing.</p><p>Here's an example of me backfilling the last 24 hours of data for our new recording rules:</p><pre><code class="language-bash">❯ kubectl exec -it prometheus-0 exec -- /bin/promtool tsdb create-blocks-from rules --start $(($(date +%s)-86400)) --end=$(date +%s) --url http://127.0.9.1:9090 /etc/prometheus/alerting.rules/anomaly.yaml
|
||
|
||
name=/etc/prometheus/alerting.rules/anomaly.yaml;anomaly.istio_request_duration_milliseconds_bucket:destination:rate1m
|
||
level=info backfiller="processing rule" id=0 name=anomaly:istio_request_duration_milliseconds_bucket:destination:rate1m:prediction
|
||
level=info backfiller="processing rule" id=1 name=anomaly:istio_request_duration_milliseconds_bucket:destination:rate1m:zscore</code></pre><p>Once this is complete; you'll need to wait for a compaction event. It'll happen pretty soon after the blocks are written, look for this in your prometheus logs:</p><pre><code class="language-bash">ts=2023-02-04T13:51:58.220Z caller=compact.go:460 level=info component=tsdb msg="compact blocks" count=3 mint=1675490400000 maxt=1675512000000 ulid=01GREB5B7VAJ5FQW8A4K298PNX sources="[01GREB59MGV9T946Z2GW798QEQ 01GREB59XTSPZ6AS6X9H3H9PFA 01GREB5AAV4A9SC51C1XFK5DCH]" duration=1.041736959s</code></pre><p>And voila, you can see here I have the last 24 hours of data populated:</p><figure class="kg-card kg-image-card"><img src="https://karlstoney.com/content/images/2023/02/Screenshot-2023-02-04-at-13.55.07.png" class="kg-image" alt="" loading="lazy" width="2000" height="716" srcset="https://karlstoney.com/content/images/size/w600/2023/02/Screenshot-2023-02-04-at-13.55.07.png 600w, https://karlstoney.com/content/images/size/w1000/2023/02/Screenshot-2023-02-04-at-13.55.07.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/02/Screenshot-2023-02-04-at-13.55.07.png 1600w, https://karlstoney.com/content/images/size/w2400/2023/02/Screenshot-2023-02-04-at-13.55.07.png 2400w" sizes="(min-width: 720px) 720px"></figure><h3 id="conclusion">Conclusion</h3><p>Math and statistics are certainly not my strong point! I learned a lot from this excellent <a href="https://about.gitlab.com/blog/2019/07/23/anomaly-detection-using-prometheus/?ref=karlstoney.com">GitLab</a> post, but statisticians please do shout up if this can be improved! But saying that I do think I've landed with a reasonable signal for outlier detection without needing to hard code specific values for individual services, which was my goal all along.</p>
|
||
</section>
|
||
|
||
<section class="article-comments gh-canvas">
|
||
|
||
<script defer src="https://cdn.jsdelivr.net/ghost/comments-ui@~1.6/umd/comments-ui.min.js" data-locale="en" data-ghost-comments="https://karlstoney.com/" data-api="https://karlstoney.com/ghost/api/content/" data-admin="https://karlstoney.com/ghost/" data-key="1fcef56573b2a5779958db5e8c" data-title="null" data-count="true" data-post-id="63de2fe2ccaa0900121710ab" data-color-scheme="auto" data-avatar-saturation="60" data-accent-color="#15171A" data-comments-enabled="all" data-publication="karlstoney.com" crossorigin="anonymous"></script>
|
||
|
||
</section>
|
||
|
||
</article>
|
||
</main>
|
||
|
||
|
||
|
||
|
||
<aside class="read-more-wrap outer">
|
||
<div class="read-more inner">
|
||
|
||
<article class="post-card post featured no-image keep-ratio">
|
||
|
||
|
||
<div class="post-card-content">
|
||
|
||
<a class="post-card-content-link" href="/from-pr-review-bot-atai-cli-to-an-autonomous-work-queue/">
|
||
<header class="post-card-header">
|
||
<div class="post-card-tags">
|
||
<span class="post-card-featured"><svg width="16" height="17" viewBox="0 0 16 17" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M4.49365 4.58752C3.53115 6.03752 2.74365 7.70002 2.74365 9.25002C2.74365 10.6424 3.29678 11.9778 4.28134 12.9623C5.26591 13.9469 6.60127 14.5 7.99365 14.5C9.38604 14.5 10.7214 13.9469 11.706 12.9623C12.6905 11.9778 13.2437 10.6424 13.2437 9.25002C13.2437 6.00002 10.9937 3.50002 9.16865 1.68127L6.99365 6.25002L4.49365 4.58752Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path>
|
||
</svg> Featured</span>
|
||
</div>
|
||
<h2 class="post-card-title">
|
||
From PR Review Bot + ATAI CLI to an Autonomous Work Queue
|
||
</h2>
|
||
</header>
|
||
<div class="post-card-excerpt">Taking our CLI, a Kubernetes Cluster and an Orchestrator to build an autonomous AI developer agent.</div>
|
||
</a>
|
||
|
||
<footer class="post-card-meta">
|
||
<time class="post-card-meta-date" datetime="2026-04-17">17 Apr 2026</time>
|
||
<span class="post-card-meta-length">8 min read</span>
|
||
<script
|
||
data-ghost-comment-count="69de00ffd764770001f6c8dd"
|
||
data-ghost-comment-count-empty=""
|
||
data-ghost-comment-count-singular="comment"
|
||
data-ghost-comment-count-plural="comments"
|
||
data-ghost-comment-count-tag="span"
|
||
data-ghost-comment-count-class-name=""
|
||
data-ghost-comment-count-autowrap="true"
|
||
>
|
||
</script>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
</article>
|
||
|
||
<article class="post-card post featured no-image keep-ratio">
|
||
|
||
|
||
<div class="post-card-content">
|
||
|
||
<a class="post-card-content-link" href="/building-atai-for-copilot/">
|
||
<header class="post-card-header">
|
||
<div class="post-card-tags">
|
||
<span class="post-card-featured"><svg width="16" height="17" viewBox="0 0 16 17" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M4.49365 4.58752C3.53115 6.03752 2.74365 7.70002 2.74365 9.25002C2.74365 10.6424 3.29678 11.9778 4.28134 12.9623C5.26591 13.9469 6.60127 14.5 7.99365 14.5C9.38604 14.5 10.7214 13.9469 11.706 12.9623C12.6905 11.9778 13.2437 10.6424 13.2437 9.25002C13.2437 6.00002 10.9937 3.50002 9.16865 1.68127L6.99365 6.25002L4.49365 4.58752Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path>
|
||
</svg> Featured</span>
|
||
</div>
|
||
<h2 class="post-card-title">
|
||
Building ATAI: Our Opinionated CLI for Copilot at Scale
|
||
</h2>
|
||
</header>
|
||
<div class="post-card-excerpt">How we wrapped Copilot, to accelerate the use of AI engineering tools at Autotrader</div>
|
||
</a>
|
||
|
||
<footer class="post-card-meta">
|
||
<time class="post-card-meta-date" datetime="2026-04-14">14 Apr 2026</time>
|
||
<span class="post-card-meta-length">8 min read</span>
|
||
<script
|
||
data-ghost-comment-count="69de1228d764770001f6c8e7"
|
||
data-ghost-comment-count-empty=""
|
||
data-ghost-comment-count-singular="comment"
|
||
data-ghost-comment-count-plural="comments"
|
||
data-ghost-comment-count-tag="span"
|
||
data-ghost-comment-count-class-name=""
|
||
data-ghost-comment-count-autowrap="true"
|
||
>
|
||
</script>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
</article>
|
||
|
||
<article class="post-card post no-image keep-ratio">
|
||
|
||
|
||
<div class="post-card-content">
|
||
|
||
<a class="post-card-content-link" href="/building-a-pr-review-agent/">
|
||
<header class="post-card-header">
|
||
<div class="post-card-tags">
|
||
</div>
|
||
<h2 class="post-card-title">
|
||
Building a PR Review Agent
|
||
</h2>
|
||
</header>
|
||
<div class="post-card-excerpt">Building a PR review agent using Gemini CLI on Kubernetes</div>
|
||
</a>
|
||
|
||
<footer class="post-card-meta">
|
||
<time class="post-card-meta-date" datetime="2025-10-08">08 Oct 2025</time>
|
||
<span class="post-card-meta-length">11 min read</span>
|
||
<script
|
||
data-ghost-comment-count="68c29ebee76f0800016b2076"
|
||
data-ghost-comment-count-empty=""
|
||
data-ghost-comment-count-singular="comment"
|
||
data-ghost-comment-count-plural="comments"
|
||
data-ghost-comment-count-tag="span"
|
||
data-ghost-comment-count-class-name=""
|
||
data-ghost-comment-count-autowrap="true"
|
||
>
|
||
</script>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
</article>
|
||
</div>
|
||
</aside>
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<footer class="site-footer outer">
|
||
<div class="inner">
|
||
<section class="copyright"><a href="https://karlstoney.com">karlstoney.com</a> © 2026</section>
|
||
<div class="site-footer-center">
|
||
<div class="site-footer-social-links">
|
||
<a href="https://x.com/karlstoney" target="_blank" rel="noopener" aria-label="X">
|
||
<svg class="icon" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M18.2439 2.25H21.5519L14.3249 10.51L22.8269 21.75H16.1699L10.9559 14.933L4.98991 21.75H1.67991L9.40991 12.915L1.25391 2.25H8.07991L12.7929 8.481L18.2439 2.25ZM17.0829 19.77H18.9159L7.08391 4.126H5.11691L17.0829 19.77Z" fill="currentColor"/>
|
||
</svg> </a>
|
||
</div>
|
||
<nav class="site-footer-nav">
|
||
<ul class="nav">
|
||
<li class="nav-twitter"><a href="https://twitter.com/karlstoney">Twitter</a></li>
|
||
<li class="nav-github"><a href="https://github.com/Stono">Github</a></li>
|
||
<li class="nav-linkedin"><a href="https://www.linkedin.com/in/karl-stoney-94995548/">LinkedIn</a></li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
<div class="gh-powered-by"><a href="https://ghost.org/" target="_blank" rel="noopener">Powered by Ghost</a></div>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">
|
||
<div class="pswp__bg"></div>
|
||
|
||
<div class="pswp__scroll-wrap">
|
||
<div class="pswp__container">
|
||
<div class="pswp__item"></div>
|
||
<div class="pswp__item"></div>
|
||
<div class="pswp__item"></div>
|
||
</div>
|
||
|
||
<div class="pswp__ui pswp__ui--hidden">
|
||
<div class="pswp__top-bar">
|
||
<div class="pswp__counter"></div>
|
||
|
||
<button class="pswp__button pswp__button--close" title="Close (Esc)"></button>
|
||
<button class="pswp__button pswp__button--share" title="Share"></button>
|
||
<button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>
|
||
<button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>
|
||
|
||
<div class="pswp__preloader">
|
||
<div class="pswp__preloader__icn">
|
||
<div class="pswp__preloader__cut">
|
||
<div class="pswp__preloader__donut"></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
|
||
<div class="pswp__share-tooltip"></div>
|
||
</div>
|
||
|
||
<button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)"></button>
|
||
<button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)"></button>
|
||
|
||
<div class="pswp__caption">
|
||
<div class="pswp__caption__center"></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<script src="/assets/built/casper.js?v=YtPhrez3GuGLVDYs" defer></script>
|
||
|
||
<script>
|
||
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
|
||
|
||
ga('create', 'UA-37161145-1', 'auto');
|
||
ga('send', 'pageview');
|
||
</script>
|
||
|
||
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"5b355c7ee12445ab8f857265175fa35d","r":1,"spa":2}' crossorigin="anonymous"></script>
|
||
</body>
|
||
</html>
|