510 lines
223 KiB
HTML
510 lines
223 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en" dir="ltr">
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
|
||
|
||
|
||
<link rel="preconnect" href="https://substackcdn.com" />
|
||
|
||
|
||
|
||
<title data-rh="true">Solving Observability's Cardinality Conundrum</title>
|
||
|
||
<meta data-rh="true" name="theme-color" content="#ffffff"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Solving Observability's Cardinality Conundrum"/><meta data-rh="true" name="twitter:title" content="Solving Observability's Cardinality Conundrum"/><meta data-rh="true" name="description" content="Introduction"/><meta data-rh="true" property="og:description" content="Introduction"/><meta data-rh="true" name="twitter:description" content="Introduction"/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!Uis6!,w_1200,h_675,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ffd726afa-fea9-4584-8816-54ee574df401_1024x540.png"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!0CKq!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fsrepath.substack.com%2Fapi%2Fv1%2Fpost_preview%2F143098468%2Ftwitter.jpg%3Fversion%3D4"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" property="article:modified_time" content="2024-04-09T12:00:33.841Z"/>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@layer legacy, pencraftReset, pencraft;
|
||
</style>
|
||
|
||
|
||
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
|
||
|
||
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/color_links.9856f2f08353fdd05959.css" />
|
||
|
||
|
||
|
||
<link rel="preload" as="font" href="https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M5knjsS_ul.woff2" crossorigin />
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.cee9fbee.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/86118.c25b9f67.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/76768.f15fc605.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/41932.97d2663c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/32552.313415d5.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
|
||
<meta name="author" content="Ash Patel" />
|
||
<meta property="og:url" content="https://read.srepath.com/p/observability-cardinality-conundrum" />
|
||
|
||
|
||
<link rel="canonical" href="https://read.srepath.com/p/observability-cardinality-conundrum" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="shortcut icon" href="https://substackcdn.com/image/fetch/$s_!rZ_-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Ffavicon.ico">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="16x16" href="https://substackcdn.com/image/fetch/$s_!6du3!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Ffavicon-16x16.png">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="32x32" href="https://substackcdn.com/image/fetch/$s_!zPo6!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Ffavicon-32x32.png">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="48x48" href="https://substackcdn.com/image/fetch/$s_!vTRX!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Ffavicon-48x48.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="57x57" href="https://substackcdn.com/image/fetch/$s_!jxVa!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-57x57.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="60x60" href="https://substackcdn.com/image/fetch/$s_!unqs!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-60x60.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="72x72" href="https://substackcdn.com/image/fetch/$s_!Un9o!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-72x72.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="76x76" href="https://substackcdn.com/image/fetch/$s_!Hvuo!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-76x76.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="114x114" href="https://substackcdn.com/image/fetch/$s_!qv6x!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-114x114.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="120x120" href="https://substackcdn.com/image/fetch/$s_!hMLj!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-120x120.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="144x144" href="https://substackcdn.com/image/fetch/$s_!sFKO!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-144x144.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="152x152" href="https://substackcdn.com/image/fetch/$s_!dkjr!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-152x152.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="167x167" href="https://substackcdn.com/image/fetch/$s_!7cpt!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-167x167.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="180x180" href="https://substackcdn.com/image/fetch/$s_!6o6g!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-180x180.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="1024x1024" href="https://substackcdn.com/image/fetch/$s_!drs4!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa72b180b-b084-408d-89dc-4c754ea5894e%2Fapple-touch-icon-1024x1024.png">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="alternate" type="application/rss+xml" href="/feed?sectionId=116798" title="Reliability Enablers"/>
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8on7mTNmnUHowCw.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onXmTNmnUHowCw.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onTmTNmnUHowCw.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onrmTNmnUHo.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M9knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M2knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M3knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M5knjsS_ul.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3FafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3OafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3PafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3BafaPWnII.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||
|
||
</style>
|
||
|
||
|
||
|
||
|
||
|
||
<style data-pub-theme-prefix>:root{--color_theme_bg_pop:#121BFA;--background_pop:#121BFA;--color_theme_bg_web:#ffffff;--cover_bg_color:#ffffff;--cover_bg_color_secondary:#f0f0f0;--background_pop_darken:#050eed;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#050eed;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(18, 27, 250, 0.2);--color_theme_bg_pop_30:rgba(18, 27, 250, 0.3);--print_pop:#121bfa;--color_theme_accent:#121bfa;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#121bfa;--home_hero:magaziney;--home_posts:list;--web_bg_color:#ffffff;--background_contrast_1:#f0f0f0;--background_contrast_2:#dddddd;--background_contrast_3:#b7b7b7;--background_contrast_4:#929292;--background_contrast_5:#515151;--color_theme_bg_contrast_1:#f0f0f0;--color_theme_bg_contrast_2:#dddddd;--color_theme_bg_contrast_3:#b7b7b7;--color_theme_bg_contrast_4:#929292;--color_theme_bg_contrast_5:#515151;--color_theme_bg_elevated:#ffffff;--color_theme_bg_elevated_secondary:#f0f0f0;--color_theme_bg_elevated_tertiary:#dddddd;--color_theme_detail:#e6e6e6;--background_contrast_pop:rgba(18, 27, 250, 0.4);--color_theme_bg_contrast_pop:rgba(18, 27, 250, 0.4);--theme_bg_is_dark:0;--print_on_web_bg_color:#363737;--print_secondary_on_web_bg_color:#868787;--background_pop_rgb:18, 27, 250;--color_theme_bg_pop_rgb:18, 27, 250;--color_theme_accent_rgb:18, 27, 250;}</style>
|
||
|
||
|
||
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
|
||
|
||
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/color_links.9856f2f08353fdd05959.css" />
|
||
|
||
|
||
<style data-pub-theme-postfix></style>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</head>
|
||
|
||
<body class="">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div id="entry">
|
||
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:71px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="flex-basis:0px;flex-grow:1;" class="logoContainer-p12gJb"><a href="/" native class="pencraft pc-display-contents pc-reset"><div draggable="false" class="pencraft pc-display-flex pc-position-relative pc-reset"><div style="width:40px;height:40px;" class="pencraft pc-display-flex pc-reset bg-white-ZBV5av pc-borderRadius-sm overflow-hidden-WdpwT6 sizing-border-box-DggLA4"><picture class="pencraft pc-display-contents pc-reset"><source type="image/webp" sizes="40px" srcset="https://substackcdn.com/image/fetch/$s_!hjhf!,w_40,h_40,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png 40w, https://substackcdn.com/image/fetch/$s_!hjhf!,w_80,h_80,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png 80w, https://substackcdn.com/image/fetch/$s_!hjhf!,w_120,h_120,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png 120w" style="display:none;"/><img style="object-fit:cover;width:40px;height:40px;" src="https://substackcdn.com/image/fetch/$s_!hjhf!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png" srcset="https://substackcdn.com/image/fetch/$s_!hjhf!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png 40w, https://substackcdn.com/image/fetch/$s_!hjhf!,w_80,h_80,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png 80w, https://substackcdn.com/image/fetch/$s_!hjhf!,w_120,h_120,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png 120w" sizes="40px" alt="Reliability Enablers" draggable="false" class="pencraft pc-reset"/></picture></div></div></a></div><div style="flex-grow:0;" class="titleContainer-DJYq5v"><h1 class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd"><a href="/" class="pencraft pc-display-contents pc-reset">Reliability Enablers</a></h1></div><div style="flex-basis:0px;flex-grow:1;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Fobservability-cardinality-conundrum&for_pub=srepath" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="height:72px;"></div></div></div><div><script nonce="igm3B5YFT1CqZB3oVHDEQw" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://read.srepath.com/p/observability-cardinality-conundrum","mainEntityOfPage":"https://read.srepath.com/p/observability-cardinality-conundrum","headline":"Solving Observability's Cardinality Conundrum","description":"Introduction","image":[{"@type":"ImageObject","url":"https://substack-post-media.s3.amazonaws.com/public/images/fd726afa-fea9-4584-8816-54ee574df401_1024x540.png"}],"datePublished":"2024-04-09T12:00:33+00:00","dateModified":"2024-04-09T12:00:33+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"Ash Patel","url":"https://substack.com/@srepath","description":"Independent reliability advocate working to drive greater software reliability beyond BigTech. Previously a seasoned operations leader.","identifier":"user:19216588","image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!-Qn7!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!-Qn7!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg"}}],"publisher":{"@type":"Organization","name":"Reliability Enablers","url":"https://read.srepath.com","description":"Soon, focus will shift to the emerging intersection of resilient and intelligent systems. Because \u201Cfive nines\u201D won\u2019t survive AI ignorance. Newsletter and podcast inside.","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":100},"identifier":"pub:2473001","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!hjhf!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!hjhf!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!hjhf!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!hjhf!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!hjhf!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!hjhf!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png"}},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":12},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":0},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":2}]}</script><script nonce="igm3B5YFT1CqZB3oVHDEQw" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Reliability Enablers","item":"https://read.srepath.com"},{"@type":"ListItem","position":2,"name":"Solving Observability's Cardinality Conundrum","item":"https://read.srepath.com/p/observability-cardinality-conundrum"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><div class="pencraft pc-display-flex pc-reset post-label"><a href="https://read.srepath.com/s/observability/?utm_source=substack&utm_medium=menu" class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-14-MLPa7j weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA">Observability focus</a></div><h1 dir="auto" class="post-title published title-X77sOw">Solving Observability's Cardinality Conundrum</h1><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-row pc-gap-8 pc-alignItems-center pc-justifyContent-flex-start pc-reset"><div style="--scale:36px;--offset:9px;--border-width:4.5px;" class="pencraft pc-display-flex pc-flexDirection-row pc-alignItems-center pc-justifyContent-flex-start pc-reset ltr-qDBmby"><a href="https://substack.com/@srepath" aria-label="View Ash Patel's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:36px;" tabindex="0" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6 avatar-u8q6xB last-JfNEJ_"><div style="--scale:36px;" title="Ash Patel" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!-Qn7!,w_36,h_36,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg 1x, https://substackcdn.com/image/fetch/$s_!-Qn7!,w_72,h_72,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg 2x, https://substackcdn.com/image/fetch/$s_!-Qn7!,w_108,h_108,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg 3x"/><img src="https://substackcdn.com/image/fetch/$s_!-Qn7!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg" alt="Ash Patel's avatar" srcset="https://substackcdn.com/image/fetch/$s_!-Qn7!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg 1x, https://substackcdn.com/image/fetch/$s_!-Qn7!,w_72,h_72,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg 2x, https://substackcdn.com/image/fetch/$s_!-Qn7!,w_108,h_108,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg 3x" width="36" height="36" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-reset color-primary-zABazT line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><span style="min-width:0;" data-state="closed"><a href="https://substack.com/@srepath" class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ">Ash Patel</a></span></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2024-04-09T12:00:33.638Z" title="2024-04-09T12:00:33.638Z">Apr 09, 2024</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (12)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">12</div></button></div><button tabindex="0" type="button" aria-label="View comments (2)" data-href="https://read.srepath.com/p/observability-cardinality-conundrum/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg><div class="label">2</div></button><button tabindex="0" type="button" id="radix-P0-27" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><h2 class="header-anchor-post">Introduction<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§introduction" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/introduction" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Cardinality is a term you’ll hear over and over again if you’re looking into how to do observability.</p><p>And especially if you are talking with vendors! They love this topic!</p><p>A lot of people have been thinking about high cardinality for a while and for a good reason.</p><p><span>Because high cardinality can cost you a LOT of money and time if you go about it wrong… but really, what we want to do is </span><strong>cut down</strong><span> </span><em><strong>excessive</strong></em><strong> cardinality</strong><span>.</span></p><p>The kind that doesn’t add value to your querying and intelligence.</p><p>That’s why dealing with cardinality is not a straightforward solution. Cutting labels willy-nilly is not the answer. We’ll get onto ways to deal with this later on.</p><p>But we should first talk about what cardinality actually means.</p><h2 class="header-anchor-post">What is cardinality?<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§what-is-cardinality" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/what-is-cardinality" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Because it’s not something you think about every day… unless you’re an observability engineer or vendor.</p><p>The last time I heard “cardinality” being used this often was in my SQL classes, which was a while back.</p><p>So I did a refresher and went down a rabbit hole of math, logic, and all that fun stuff.</p><p><span>Cardinality refers to </span><em><strong>how many unique values there are in a data set.</strong></em></p><p>You’re essentially looking for how diverse — and ultimately complex — the data is.</p><h3 class="header-anchor-post">How do you differentiate between low and high cardinality data?<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§how-do-you-differentiate-between-low-and-high-cardinality-data" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/how-do-you-differentiate-between-low-and-high-cardinality-data" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p>Low cardinality means low complexity with few dimensions to data. This is fine for analyzing aggregate data but lacks granularity that engineers often seek to solve system problems</p></li><li><p>High cardinality contains more data dimensions. This lets you slice and dice data for more detailed analysis, but you then have to deal with complexity issues.</p></li></ul><p>By dimensions, I mean attributes like method, status_code, instance_id, etc.</p><p><strong>Let’s run through a simple example to cement the difference:</strong></p><p><span>Let’s say you have the table </span><em>fruits</em><span> with only the fruit </span><em>apple</em><span> in your database. We want to add a </span><em>color</em><span> key to identify each apple in your table.</span></p><p>Now, the data on apples is low cardinality if you only find red or green apples like so:</p><p>But it becomes higher cardinality if you have apples with colors like orange, blue, red, pink, green, violet, black etc.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!ZNbn!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!ZNbn!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 424w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 848w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 1272w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!ZNbn!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":null,"width":null,"resizeWidth":null,"bytes":null,"alt":"Low vs high cardinality data example visual","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt="Low vs high cardinality data example visual" title="Low vs high cardinality data example visual" srcset="https://substackcdn.com/image/fetch/$s_!ZNbn!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 424w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 848w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 1272w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p><span>The more color options you have for </span><em>apple</em><span>, the higher the cardinality of the </span><em>color</em><span> data.</span></p><h3 class="header-anchor-post">Examples of high cardinality<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§examples-of-high-cardinality" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/examples-of-high-cardinality" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Here are some examples of high cardinality data:</p><ul><li><p>email addresses (never append these to metrics!)</p></li><li><p><span>user IDs (</span><em>”Observability engineer: why is this in my beautiful TSDB?!”</em><span>)</span></p></li><li><p>IP addresses (sometimes appended for AppSec purposes)</p></li><li><p>instance name (fair use case for identifying instance issues)</p></li><li><p>pod s (like the kind you find within Kubernetes)</p></li></ul><h2 class="header-anchor-post">3 key benefits of high cardinality in observability data<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§3-key-benefits-of-high-cardinality-in-observability-data" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/3-key-benefits-of-high-cardinality-in-observability-data" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>High cardinality enables these traits within observability data:</p><h3 class="header-anchor-post">1. Granularity<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§1-granularity" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/1-granularity" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>With granular data, you can slice and dice to deeper and deeper levels to precisely pinpoint where issues like performance degradation and outages are happening.</p><p><span>For example, with each </span><code>500 error</code><span>, you may want to dig a little further.</span></p><p><span>With a high cardinality metric, you can dig into the 50x for </span><code>GET</code><span>, </span><code>POST</code><span>, and </span><code>DELETE</code><span> requests.</span></p><h3 class="header-anchor-post">2. Segmentation<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§2-segmentation" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/2-segmentation" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Helps you collect disparate data and organize it into chunks for easier human processing.</p><p>Examples include:</p><ul><li><p>payments by age group e.g. 18-25, 25-34, 35-49, etc.</p></li><li><p>payments by market region e.g. North America, South East Asia, LatAm, EU, etc.</p></li></ul><h3 class="header-anchor-post">3. Performance evaluation<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§3-performance-evaluation" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/3-performance-evaluation" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>High cardinality means you can go beyond “success” and “fail” for responses.</p><p>You can classify it as performance levels to improve reliability for different instances and groups.</p><p>Imagine what you can work out by putting responses into buckets of 0-10 scoring.</p><p><strong>Before we continue, I will assume that you know the answers to the following questions:</strong></p><p>What does observability data look like?</p><p>How does observability data flow?</p><p>Where observability data gets stored and how</p><h2 class="header-anchor-post">How do observability data types do in terms of cardinality?<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§how-do-observability-data-types-do-in-terms-of-cardinality" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/how-do-observability-data-types-do-in-terms-of-cardinality" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Metrics are the main data type when we think about cardinality. But let’s still cover each of the 3 main observability data types to see its cardinality issue:</p><h3 class="header-anchor-post"><span>High cardinality in l</span><strong>ogs</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§high-cardinality-in-logs" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/high-cardinality-in-logs" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Logs of yesteryear are typically less affected by high cardinality than metrics and traces.</p><p>Some engineers still use unstructured logs for small-scale systems.</p><p>At the commercial scale, modern logs are structured or at least shifting toward that. JSON shows the most promise through that.</p><p>So why shift to a structured format? A few things come into play like:</p><ul><li><p>Readability</p></li><li><p>Better ability to group or segment logs by keys for better more refined analytical outcomes</p></li><li><p>Speed to read as less string manipulation or string processing has to take place during the reading of the aggregated log data</p></li></ul><p>You can derive metrics from logs now!</p><p>Logs in a commercial setting can experience high cardinality, just like metrics and traces.</p><h3 class="header-anchor-post"><span>High cardinality in m</span><strong>etrics</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§high-cardinality-in-metrics" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/high-cardinality-in-metrics" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Among the trio, metrics are the most significantly affected by high cardinality.</p><p>Metrics get complicated by the fact that they can have many dimensions.</p><p>You can have a multiplier effect when you add a new dimension.</p><p>This multiplication is what defines the high cardinality we are talking about.</p><p>You benefit in one way from this through more granularity that helps deeper analysis.</p><p>But then you also impact storage, querying, and visualization performance.</p><h3 class="header-anchor-post"><span>High cardinality in t</span><strong>races</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§high-cardinality-in-traces" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/high-cardinality-in-traces" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Traces can also be affected by high cardinality.</p><p>Picture a trace to be the bus route from New York to Los Angeles and back.</p><p>This route can be divided into sections — when you plan a trip along this route you may want to stop for a breather, food, or a bathroom break.</p><p>Each section of this route or round trip is equivalent to what in tracing we would call a span.</p><p>A common dimension for spans is the time or duration of that specific section of work.</p><p>We can add more detail or context for each span by appending metadata to the span.</p><p>You get high cardinality within tracing when you use spans like user IDs and specific names of what happened.</p><p>All this once again impacts storage and querying performance.</p><h2 class="header-anchor-post">Calculating the impact of cardinality within a metric<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§calculating-the-impact-of-cardinality-within-a-metric" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/calculating-the-impact-of-cardinality-within-a-metric" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Adding a dimension to a metric does not cause high cardinality.</p><p>It’s what that key-value pair stands for that determines this.</p><p>So adding a 6th dimension doesn’t increase cardinality if it’s a boolean or a “success”/”fail” type.</p><p>But if it were something like instance ID and you have 100 or 1000 instances, that would.</p><p>We do not need to calculate cardinality unless we are sitting in a math class.</p><p>We are more interested in how many time series</p><h3 class="header-anchor-post">Calculating time series count for a metric<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§calculating-time-series-count-for-a-metric" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/calculating-time-series-count-for-a-metric" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Let’s run through a simple example.</p><p><span>Say you have a metric called </span><em>network_latency_distribution</em><span> covering 100 instances with 10 buckets, 10 possible response codes, and 10 network paths.</span></p><pre><code>**Calculating the series would look like this:**
|
||
= 100 instances * 10 buckets * 10 response codes * 10 paths
|
||
= 100,000 series
|
||
</code></pre><p>This is a reasonable size for a series, but things can get out of hand as you add more dimensions.</p><p><strong>Adding a dimension like </strong><em><strong>region</strong></em><strong> can significantly increase cardinality.</strong></p><pre><code>**Say we have 6 regions to choose from. The calculation would become:**
|
||
= 100 instances x 10 buckets x 10 response codes x 10 paths x 6 regions
|
||
= 600,000 series (! 😕)
|
||
</code></pre><p>Okay. It doesn’t look excessive, but can significantly increase querying time!</p><p>I’ll share some query time data with you in a minute.</p><p><strong>The dimension of </strong><em><strong>pods</strong></em><strong> (within containers) can increase cardinality by a huge factor, too.</strong></p><pre><code>**Say we have a low 1,000 pods in action. The calculation would become:**
|
||
= 100 instances x 10 buckets x 10 response codes x 10 paths x 1,000 pods
|
||
= 100,000,000 series (!! 😦)
|
||
</code></pre><p><strong>Excessive cardinality comes into the picture when you add a dimension like user_id.</strong></p><pre><code>**Even with a modest user count of 10,000, the time series blows out with:**
|
||
= 100 instances * 10 buckets * 10 response codes * 10 paths * 10000 users
|
||
= 1,000,000,000 series (!!! 😱)
|
||
</code></pre><p>We’ll talk about Excessive cardinality in more detail later on.</p><p><span>🚨 </span><strong>PSA:</strong><span> The more types of data you collect, the more fragmented view you have of the whole piece. And if you have a fragmented view, that reduces your ability to TAKE ACTION on the data. Remember that </span><strong>Quality of data > Quantity of data</strong><span>.</span></p><h2 class="header-anchor-post">High cardinality data is rising because of system trends<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§high-cardinality-data-is-rising-because-of-system-trends" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/high-cardinality-data-is-rising-because-of-system-trends" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><h3 class="header-anchor-post">Shift from monolith to microservices architecture<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§shift-from-monolith-to-microservices-architecture" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/shift-from-monolith-to-microservices-architecture" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Where you had one humongous service emitting metrics, you now have 10, 20, 50, 100+ microservices with each emitting its metrics. ‘Nuf said.</p><h3 class="header-anchor-post">Shift from VM-based to container-based infrastructure<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§shift-from-vm-based-to-container-based-infrastructure" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/shift-from-vm-based-to-container-based-infrastructure" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Who would’ve thought that life was easier when we depended on VM infrastructure rather than containers? </sarcasm></p><p>Containers have their benefits but generate a whole bunch of time-series data.</p><p>For every 1 VM in an older system, there are 10s of containers to match workloads.</p><p><span>The key culprit behind this is their </span><em><strong>high ephemerality</strong></em><span>.</span></p><p>The ephemeral nature of containers refers to when they start, shut down and then a new container takes their place.</p><p>All this stop-start emits a lot of data for metrics collectors to ingest and push to storage.</p><h3 class="header-anchor-post">Serverless functions as a large part of the system<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§serverless-functions-as-a-large-part-of-the-system" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/serverless-functions-as-a-large-part-of-the-system" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Every time you invoke a Lambda function, that induces a metric with a time series. Depending on the particular service you are running on serverless, this time series number can be HUGE.</p><p>A whole bunch of factors come into play like:</p><ol><li><p>how often you're invoking the lambda function</p></li><li><p>the number of serverless functions in your system</p></li><li><p>concurrency of handling requests and scale</p></li></ol><p>Your time series data incurs higher and higher cardinality as these 3 factors rise in frequency or occurrence (depending on the factor).</p><h2 class="header-anchor-post">When to deal with high cardinality<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§when-to-deal-with-high-cardinality" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/when-to-deal-with-high-cardinality" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Remember how observability data flows. There are 4 distinct stages:</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!PUeG!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!PUeG!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 424w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 848w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 1272w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!PUeG!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":null,"width":null,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt title srcset="https://substackcdn.com/image/fetch/$s_!PUeG!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 424w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 848w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 1272w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p><span>Dealing with high cardinality at the </span><em>Instrumentation</em><span> stage is not ideal. You will not confidently know what dimensions to filter out at this stage.</span></p><p><span>It’s also not ideal at the </span><em>Usage</em><span> stage. It’s too late to deal with cardinality issues because you’ve incurred high ingestion and storage costs, so…</span></p><p><span>Deal with high cardinality at the </span><em>Ingestion & Storage</em><span> stages. This is when you can put in practices like in-flight aggregation, cardinality isolation, and cardinality limiters.</span></p><h2 class="header-anchor-post">When to ditch high cardinality data<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§when-to-ditch-high-cardinality-data" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/when-to-ditch-high-cardinality-data" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>High cardinality data is a necessary evil in some situations. As I mentioned earlier, it helps with granularity, segmentation, and deeper performance evaluation.</p><p>If it does not add value to your querying, intelligence, or alerting. Ditch it.</p><p>It’s just an ornament to give you pretty data.</p><p><span>What I mean by this is it </span><em>looks</em><span> like it </span><em>could</em><span> be important.</span></p><p>But in reality:</p><ul><li><p>it’s giving you limited analytical value</p></li><li><p><span>rarely contributes to usable insight </span><strong>and</strong></p></li><li><p>with either of the above two is resource intensive.</p></li></ul><p>Because if resource intensiveness did not cost money or time, we wouldn’t care about it.</p><p>We’d just let high cardinality data sit there and do its thing.</p><p>But it does cost money in terms of storage and processing power.</p><p>And it costs time in terms of how long you have to wait before you can start working with the data.</p><p>And it’s definitively something you need to look into if it’s slowing things down to the point where you’re pushing past your MTTR target.</p><p>(MTTR = your mean time to recovery, repair, resolution, whatever you want to call it).</p><p>Now, you might be saying thank you, Captain Obvious.</p><p>But think about how often this is still a real problem in software systems.</p><p>You need to pose it as a challenge for people to think about and solve it.</p><h2 class="header-anchor-post"><span>The problem with </span><em>excessive</em><span> cardinality</span><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§the-problem-with-excessive-cardinality" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/the-problem-with-excessive-cardinality" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>The working group behind the open-source Prometheus monitoring tool has warned about this for a while.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!PQcc!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!PQcc!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 424w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 848w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 1272w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!PQcc!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":null,"width":null,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt title srcset="https://substackcdn.com/image/fetch/$s_!PQcc!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 424w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 848w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 1272w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p><span>Excerpt from </span><a href="http://Prometheus.io">Prometheus.io</a><span> documentation on </span><a href="https://prometheus.io/docs/practices/naming/">Metric and label naming</a></p><p>A quick TLDR of their alert message on key-value pairs:</p><blockquote><p>“Every unique combination of key-value pairs represents a new time-series. This significantly increases the data stored.”</p></blockquote><p>An ever-increasing number of possible key-value pairs or dimensions does something sinister within time series databases (TSDBs).</p><p>The number of series for a single metric will explode 💥 to the point that your querying eventually slows down to a crash.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!0EGq!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!0EGq!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 424w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 848w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 1272w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!0EGq!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":null,"width":null,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt title srcset="https://substackcdn.com/image/fetch/$s_!0EGq!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 424w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 848w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 1272w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p>Query performance data by Chris Marchbanks (ex-Splunk, Grafana Labs) says it all.</p><p>This is how long it took to query quantities of time series:</p><ul><li><p>100,000 series took 1.5 seconds (acceptable)</p></li><li><p>200,000 series took 5 seconds (slow-ish)</p></li><li><p>10,000,000 series took 15 minutes (!)</p></li></ul><p><span>Keep this in mind: high or (better put) </span><strong>excessive cardinality is a data problem at its core</strong><span>.</span></p><p>Not only does it slow down query time to a grind, but it can also cause trouble like:</p><ul><li><p>data being wasted on dimensions that are not needed for system improvements</p></li><li><p>need for engineers to constantly maintain performance of the observability system</p></li><li><p>higher storage needs and processing resources means higher cost to run</p></li></ul><p>We will think through a few solution starters toward the end to prevent or at least reduce this risk.</p><h2 class="header-anchor-post">What’s contributing to excessive cardinality in a software system?<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§whats-contributing-to-excessive-cardinality-in-a-software-system" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/whats-contributing-to-excessive-cardinality-in-a-software-system" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><h3 class="header-anchor-post">Bad dimension selection<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§bad-dimension-selection" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/bad-dimension-selection" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Remember my calculation example above?</p><p>Putting in user or request IDs can skyrocket your observability metrics’ cardinality.</p><p>Each unique identifier will contribute to a new entry, increasing the overall cardinality of the logs.</p><h3 class="header-anchor-post">Improper sample practices<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§improper-sample-practices" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/improper-sample-practices" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Observability systems generate a ton of data all the time.</p><p>It’s not an easy task to query all of that data all the time.</p><p>This is where you want to bring sampling practices to select a portion of the data to analyze.</p><p>But if you don’t use the right sampling techniques, you will deal with high cardinality data and more series than your dashboard or query tool can handle.</p><h3 class="header-anchor-post">Unbounded event types<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§unbounded-event-types" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/unbounded-event-types" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Metrics are not the sole culprits for pushing out excessive cardinality data.</p><p>Putting weak boundaries around your event data can do the same to logs and traces.</p><p>A good system will have few event types while a system suffering from excessive cardinality will have numerous possible event types.</p><p>The more event types there are, the more logs and spans you’ll have to push to ingestion.</p><p>This is not an exhaustive list of ways excessive cardinality can happen. I want to illustrate the idea that there are several ways you can end up with it.</p><h2 class="header-anchor-post"><span>How to solve </span><em>excessive</em><span> cardinality</span><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§how-to-solve-excessive-cardinality" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/how-to-solve-excessive-cardinality" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><h3 class="header-anchor-post">Be selective about dimensions<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§be-selective-about-dimensions" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/be-selective-about-dimensions" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>I feel like I’ve mentioned it several times already, but never, ever use dimensions like email address, user ID, transaction ID, or anything with overly unique data in your metrics.</p><h3 class="header-anchor-post">Split the metric into smaller metrics<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§split-the-metric-into-smaller-metrics" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/split-the-metric-into-smaller-metrics" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Ask yourself these two questions:</p><blockquote><p>Do I need to have this single metric with all these dimensions?</p></blockquote><blockquote><p>Can I split it into two separate metrics that can still help me answer the questions I will pose at querying, and still give me the necessary alerting?</p></blockquote><p>An example might better highlight why you’d want to do this:</p><p><span>Let’s return to our metric called </span><em>network_latency_distribution, which</em><span> covers 100 instances with 10 buckets, 10 possible response codes, and 10 network paths.</span></p><pre><code>**Calculating the series would look like this:**
|
||
= 100 instances * 10 buckets * 10 response codes * 10 paths
|
||
= 100,000 series
|
||
</code></pre><p>Now what would happen if we were to split this metric into 2 individual metrics, one without paths and one without buckets?</p><pre><code>**Here's the first metric without paths:**
|
||
= 100 instances * 10 buckets * 10 response codes
|
||
= 10,000 series
|
||
</code></pre><pre><code>**Here's the second metric without buckets:**
|
||
= 100 instances * 10 response codes * 10 paths
|
||
= 10,000 series
|
||
</code></pre><p><span>This gives us a grand total of 20,000 series and </span><strong>a whopping 80,000 series reduction</strong><span>!</span></p><p>This split works perfectly if we don’t need to correlate paths with buckets to solve system issues.</p><p>How much faster would it be to query 20,000 vs 100,000 series?</p><p>Answer: enough to feel instantaneous vs. time spent hearing, “Please wait while we process your query.”</p><h3 class="header-anchor-post">Allow high(er) cardinality for high-value metrics only<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§allow-higher-cardinality-for-high-value-metrics-only" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/allow-higher-cardinality-for-high-value-metrics-only" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>It can still make sense to generate a whole bunch of series for metrics that add business value.</p><p>You may have some metrics that need 100,000 series rather than being split into multiple mini-metrics and losing their strength.</p><p>How do we define a high-value metric? It is this if it supports:</p><ul><li><p>critical decision-making processes</p></li><li><p>developing actionable insights or</p></li><li><p>enhancement of overall system performance</p></li></ul><p>What hits one of these criteria depends on your software architecture, industry context, and the problems you’re looking to solve.</p><p>Here are a few considerations to make:</p><ol><li><p>These kinds of metrics might not be suitable for time-of-essence needs like alerting</p></li><li><p>You can try and save the designation of high-value metrics to ad-hoc queries, low-usage dashboards for specialist or special interest groups, and periodic reports</p></li><li><p>You still need to weigh the impact vs querying & visualization time</p></li><li><p>Even a high-value metric can start looking too expensive with the way observability is priced these days, so work out the costings vs the value you attain and discuss with management</p></li></ol><p>We must still remember to control cardinality levels to meet our cost and time-to-productivity constraints.</p><h2 class="header-anchor-post">Wrapping up<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§wrapping-up" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.srepath.com/i/143098468/wrapping-up" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>If you can only remember one thing from this guide, I want it to be this 👇🏼</p><p><strong>Cardinality is valuable, but excessive cardinality is expensive — in terms of time to query, cost to store, and resource consumption to process & analyze. Keep it in check.</strong></p></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (12)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">12</div></button></div><button tabindex="0" type="button" aria-label="View comments (2)" data-href="https://read.srepath.com/p/observability-cardinality-conundrum/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg><div class="label">2</div></button><button tabindex="0" type="button" id="radix-P0-56" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="visibility-check"></div><div id="discussion" class="pencraft pc-display-flex pc-flexDirection-column pc-gap-16 pc-paddingTop-32 pc-paddingBottom-32 pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-32 pc-reset container"><h4 class="pencraft pc-reset line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Discussion about this post</h4><div class="pencraft pc-alignSelf-flex-start pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Select discussion type" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-70-content-comments" data-state="active" id="radix-P0-70-trigger-comments" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Comments</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-70-content-restacks" data-state="inactive" id="radix-P0-70-trigger-restacks" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Restacks</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div></div></div><div id="substack-comments" class="single-post-section comments-section"><div class="container"><div class="visibility-check"></div><div data-test-id="comment-input" class="pencraft pc-display-flex pc-reset flex-grow-rzmknG"><form class="comment-input-form form-CkZ7Kt"><div style="--scale:32px;" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><div style="--scale:32px;" title="User" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-flexDirection-column pc-boxShadow-xs pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-sm"><textarea name="body" placeholder="Write a comment..." aria-label="Write a comment..." rows="4" class="pencraft input-qHk4bN autogrowing-_ipn9Y textarea-GbEjRX inputText-pV_yWb unstyled-IJLUJU"></textarea></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></form></div><div class="comment-list post-page-root-comment-list"><div class="comment-list-items"><div class="comment"><div id="comment-55063338" class="comment-anchor"></div><div id="comment-55063338-reply" class="comment-anchor"></div><div role="article" aria-label="Comment by Mark Seery" class="pencraft pc-display-flex pc-gap-12 pc-paddingBottom-12 pc-reset comment-content"><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><a href="https://substack.com/profile/16078870-mark-seery?utm_source=comment" aria-label="View Mark Seery's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:32px;" tabindex="0" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6"><div style="--scale:32px;" title="Mark Seery" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!Glqo!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa81e1337-ef00-4d85-9248-5707e7494ad5_800x800.png 1x, https://substackcdn.com/image/fetch/$s_!Glqo!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa81e1337-ef00-4d85-9248-5707e7494ad5_800x800.png 2x, https://substackcdn.com/image/fetch/$s_!Glqo!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa81e1337-ef00-4d85-9248-5707e7494ad5_800x800.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!Glqo!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa81e1337-ef00-4d85-9248-5707e7494ad5_800x800.png" alt="Mark Seery's avatar" srcset="https://substackcdn.com/image/fetch/$s_!Glqo!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa81e1337-ef00-4d85-9248-5707e7494ad5_800x800.png 1x, https://substackcdn.com/image/fetch/$s_!Glqo!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa81e1337-ef00-4d85-9248-5707e7494ad5_800x800.png 2x, https://substackcdn.com/image/fetch/$s_!Glqo!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa81e1337-ef00-4d85-9248-5707e7494ad5_800x800.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-4 pc-reset"><div class="pencraft pc-display-flex pc-minWidth-0 pc-gap-8 pc-alignItems-center pc-justifyContent-space-between pc-reset line-height-20-t4M0El font-text-qe4AeH size-15-Psle70 weight-regular-mUq6Gb"><div class="pencraft pc-display-flex pc-minWidth-0 pc-gap-8 pc-alignItems-center pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-gap-6 pc-reset color-primary-zABazT line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb reset-IxiVJZ comment-author-name"><div class="pencraft pc-display-flex pc-gap-4 pc-minWidth-0 pc-alignItems-center pc-reset"><span class="pencraft pc-reset ellipsis-Jb8om8 line-height-20-t4M0El font-text-qe4AeH size-15-Psle70 weight-medium-fw81nC reset-IxiVJZ"><span style="min-width:0;" data-state="closed"><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a href="https://substack.com/profile/16078870-mark-seery?utm_source=substack-feed-item" showBack data-native="true" class="link-LIBpto">Mark Seery</a></span></span></span></div></div><span style="min-width:0;" data-state="closed"></span><a data-native="true" href="https://read.srepath.com/p/observability-cardinality-conundrum/comment/55063338" rel="nofollow" title="Apr 29, 2024, 3:57 AM" class="pencraft pc-reset color-secondary-ls1g8s decoration-hover-underline-ClDVRM reset-IxiVJZ"><span class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb nowrap-QngyoB reset-IxiVJZ">Apr 29, 2024</span></a></div></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><button tabindex="-1" type="button" class="pencraft pc-display-flex pc-gap-4 pc-height-20 pc-paddingLeft-6 pc-paddingRight-6 pc-paddingTop-2 pc-paddingBottom-2 pc-alignItems-center pc-reset font-text-qe4AeH size-11-NuY2Zx weight-medium-fw81nC pencraft tag-XbOVLt theme_accent-Y2sqZY priority_secondary-outline-RpooJS pencraft pc-display-flex pc-height-20 pc-paddingLeft-6 pc-paddingRight-6 pc-paddingTop-2 pc-paddingBottom-2 pc-gap-4 pc-alignItems-center pc-reset cursor-inherit-LxLBJ6 pc-borderRadius-xs font-text-qe4AeH size-11-NuY2Zx weight-medium-fw81nC"><div class="pencraft pc-display-flex pc-alignItems-center pc-reset leading-mI5Ihl fillIcon-dQ0mii"><svg aria-hidden="true" style="height:14px;width:14px;" width="14" height="14" viewBox="0 0 20 20" fill="var(--color-fg-primary)" stroke-width="2.5" stroke="#000" xmlns="http://www.w3.org/2000/svg"><g><path stroke="none" d="M9.99915 16.7256C9.90515 16.7256 9.79102 16.692 9.65674 16.6249C9.52246 16.5622 9.3949 16.4906 9.27405 16.41C8.02974 15.6044 6.94657 14.7584 6.02454 13.8722C5.10697 12.9815 4.3953 12.0662 3.88953 11.1262C3.38375 10.1818 3.13086 9.23067 3.13086 8.27283C3.13086 7.63725 3.23157 7.05762 3.43298 6.53394C3.63888 6.01025 3.92086 5.55819 4.27893 5.17773C4.64148 4.79728 5.05774 4.50635 5.52771 4.30493C6.00216 4.09904 6.51241 3.99609 7.05847 3.99609C7.73433 3.99609 8.31844 4.16618 8.81079 4.50635C9.30762 4.84652 9.70374 5.28963 9.99915 5.83569C10.299 5.28516 10.6951 4.84204 11.1875 4.50635C11.6843 4.16618 12.2707 3.99609 12.9465 3.99609C13.4836 3.99609 13.9894 4.09904 14.4639 4.30493C14.9428 4.50635 15.3613 4.79728 15.7194 5.17773C16.0774 5.55819 16.3572 6.01025 16.5586 6.53394C16.7645 7.05762 16.8674 7.63725 16.8674 8.27283C16.8674 9.23067 16.6145 10.1818 16.1088 11.1262C15.603 12.0662 14.8891 12.9815 13.967 13.8722C13.0495 14.7584 11.9708 15.6044 10.731 16.41C10.6056 16.4906 10.4758 16.5622 10.3416 16.6249C10.2118 16.692 10.0976 16.7256 9.99915 16.7256Z"></path></g></svg></div>Liked by Ash Patel</button></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"></div><div class="pencraft pc-display-flex pc-reset comment-menu-trigger-container triggerContainer-eX588u"><button tabindex="0" type="button" aria-label="Ellipsis" id="radix-P0-83" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft trigger-j08Uop iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_quaternary-kpMibu"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-ellipsis"><circle cx="12" cy="12" r="1"></circle><circle cx="19" cy="12" r="1"></circle><circle cx="5" cy="12" r="1"></circle></svg></button></div></div><div class="comment-body expanded"><p><span>Great post!! Thanks.</span></p></div><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-8 pc-justifyContent-flex-start pc-alignItems-center pc-reset comment-actions withShareButton-hQzuEn"><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a class="pencraft pc-reset link-_X6et2 link-LIBpto"><div class="pencraft pc-display-flex pc-gap-6 pc-alignItems-center pc-reset"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="var(--color-fg-secondary-themed)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA comment-actions-label">Reply</div></div></a></span><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a class="pencraft pc-reset link-_X6et2 link-LIBpto"><div class="pencraft pc-display-flex pc-gap-6 pc-alignItems-center pc-reset"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="var(--color-fg-secondary-themed)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA comment-actions-label">Share</div></div></a></span></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></div><div class="more-replies-container"><a href="https://read.srepath.com/p/observability-cardinality-conundrum/comment/55063338" class="more-replies">1 reply by Ash Patel</a></div></div></div></div><a href="https://read.srepath.com/p/observability-cardinality-conundrum/comments" class="more-comments">1 more comment...</a></div></div></div><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-101-content-top" data-state="active" id="radix-P0-101-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-101-content-new" data-state="inactive" id="radix-P0-101-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-101-content-community" data-state="inactive" id="radix-P0-101-trigger-community" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Discussions</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer themed-background"><div class="container"><div class="footer-terms"><span>© 2026 Ash P</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&utm_medium=web&utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="igm3B5YFT1CqZB3oVHDEQw">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
|
||
</div>
|
||
|
||
|
||
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
|
||
<script nonce="igm3B5YFT1CqZB3oVHDEQw">
|
||
window.Sentry && window.Sentry.onLoad(function() {
|
||
window.Sentry.init({
|
||
environment: window._preloads.sentry_environment,
|
||
dsn: window._preloads.sentry_dsn,
|
||
})
|
||
})
|
||
</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script nonce="igm3B5YFT1CqZB3oVHDEQw">window._preloads = JSON.parse("{\"cspNonce\":\"igm3B5YFT1CqZB3oVHDEQw\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://read.srepath.com\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":19216588,\"byline_images_enabled\":true,\"bylines_enabled\":true,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Ash P\",\"cover_photo_url\":null,\"created_at\":\"2024-03-30T10:17:21.113Z\",\"custom_domain_optional\":false,\"custom_domain\":\"read.srepath.com\",\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":null,\"default_show_guest_bios\":true,\"email_banner_url\":null,\"email_from_name\":\"Ash Patel\",\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":null,\"free_subscription_benefits\":null,\"ga_pixel_id\":null,\"google_site_verification_token\":null,\"google_tag_manager_token\":null,\"hero_image\":null,\"hero_text\":\"Soon, focus will shift to the emerging intersection of resilient and intelligent systems. Because \u201Cfive nines\u201D won\u2019t survive AI ignorance. Newsletter and podcast inside.\",\"hide_intro_subtitle\":null,\"hide_intro_title\":null,\"hide_podcast_feed_link\":false,\"homepage_type\":\"magaziney\",\"id\":2473001,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":null,\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!hjhf!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png\",\"minimum_group_size\":2,\"moderation_enabled\":true,\"name\":\"Reliability Enablers\",\"paid_subscription_benefits\":null,\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"disabled\",\"paywall_free_trial_enabled\":false,\"podcast_art_url\":null,\"paid_podcast_episode_art_url\":null,\"podcast_byline\":null,\"podcast_description\":null,\"podcast_enabled\":false,\"podcast_feed_url\":null,\"podcast_title\":null,\"post_preview_limit\":null,\"primary_user_id\":19216588,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"srepath\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#121BFA\",\"theme_var_color_links\":true,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":null,\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":null,\"stripe_user_id\":null,\"stripe_country\":null,\"stripe_publishable_key\":null,\"stripe_platform_account\":null,\"automatic_tax_enabled\":null,\"author_name\":\"Ash Patel\",\"author_handle\":\"srepath\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!-Qn7!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg\",\"author_bio\":\"Independent reliability advocate working to drive greater software reliability beyond BigTech. Previously a seasoned operations leader.\",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":null,\"web_bg_color\":\"#ffffff\",\"cover_bg_color\":\"#ffffff\",\"publication_id\":2473001,\"color_links\":null,\"font_preset_heading\":null,\"font_preset_body\":null,\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"magaziney\",\"home_posts\":\"list\",\"home_show_top_posts\":false,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":null,\"custom_footer\":null,\"social_media_links\":null,\"font_options\":null,\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":null,\"default_group_coupon_percent_off\":null,\"default_group_coupon_include_founding\":null,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2021-08-31T22:07:00.000Z\",\"has_podcast\":true,\"has_free_podcast\":true,\"has_subscriber_only_podcast\":false,\"has_community_content\":true,\"rankingDetail\":\"Launched 2 years ago\",\"rankingDetailFreeIncluded\":\"Hundreds of subscribers\",\"rankingDetailOrderOfMagnitude\":0,\"rankingDetailFreeIncludedOrderOfMagnitude\":100,\"rankingDetailFreeSubscriberCount\":null,\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u062A\u0645 \u0627\u0644\u0625\u0637\u0644\u0627\u0642 2 years ago\"},\"ca\":{\"rankingDetail\":\"S\u2019ha llan\u00E7at fa 2 anys\"},\"da\":{\"rankingDetail\":\"Lancering 2 \u00E5r\"},\"de\":{\"rankingDetail\":\"vor 2 Jahren gelauncht\"},\"es\":{\"rankingDetail\":\"Lanzado hace 2 a\u00F1os\"},\"fr\":{\"rankingDetail\":\"Lanc\u00E9 il y a 2 ann\u00E9es\"},\"id\":{\"rankingDetail\":\"Diluncurkan 2 tahun yang lalu\"},\"ja\":{\"rankingDetail\":\"\u958B\u59CB\u65E5 2\u5E74\u524D\"},\"ko\":{\"rankingDetail\":\"2\uB144 \uC804\uC5D0 \uCD9C\uC2DC\"},\"nb\":{\"rankingDetail\":\"Lansert 2 \u00E5r\"},\"nl\":{\"rankingDetail\":\"Gelanceerd 2 jaar geleden\"},\"pl\":{\"rankingDetail\":\"Uruchomiono 2 lat temu\"},\"pt\":{\"rankingDetail\":\"Lan\u00E7ado 2 anos\"},\"pt-br\":{\"rankingDetail\":\"Lan\u00E7ado 2 anos\"},\"en-gb\":{\"rankingDetail\":\"Launched 2 years ago\"},\"it\":{\"rankingDetail\":\"Lanciato 2 anni\"},\"tr\":{\"rankingDetail\":\"2 y\u0131l ba\u015Flat\u0131ld\u0131\"},\"sv\":{\"rankingDetail\":\"Lanserad 2 \u00E5r sedan\"},\"fi\":{\"rankingDetail\":\"Launched 2 vuotta\"},\"is\":{\"rankingDetail\":\"Launched 2 \u00E1r\"},\"zh-hans\":{\"rankingDetail\":\"\u4E8E2 \u5E74\u524D\u4E0A\u7EBF\"},\"zh-hant\":{\"rankingDetail\":\"\u65BC 2 \u5E74\u524D \u63A8\u51FA\"},\"en\":{\"rankingDetail\":\"Launched 2 years ago\"}},\"freeSubscriberCount\":null,\"freeSubscriberCountOrderOfMagnitude\":\"831\",\"author_bestseller_tier\":0,\"author_badge\":null,\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":false,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":null,\"primary_profile_name\":\"Ash Patel\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!-Qn7!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg\",\"no_follow\":false,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"free\",\"sections\":[{\"id\":116798,\"created_at\":\"2024-04-01T07:43:46.582Z\",\"updated_at\":\"2024-04-01T07:52:50.349Z\",\"publication_id\":2473001,\"name\":\"Observability focus\",\"description\":\"Covering concepts related to observability (aka o11y)\",\"slug\":\"observability\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":1,\"port_status\":\"success\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/b2430b5d-ad76-4ca7-99d3-ba22adc5bd1a_1280x1280.png\",\"hide_from_navbar\":false,\"email_from_name\":\"Ash Patel\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":120870,\"created_at\":\"2024-04-22T15:22:41.075Z\",\"updated_at\":\"2024-05-31T04:49:53.399Z\",\"publication_id\":2473001,\"name\":\"Hear the Podcast\",\"description\":\"Audio conversations around reliability, especially around enabling it across teams and organizations\",\"slug\":\"podcast\",\"is_podcast\":true,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":2,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":{\"section_id\":120870,\"itunes_podcast_primary_category\":\"Technology\",\"itunes_podcast_primary_subcategory\":\"\",\"itunes_podcast_secondary_category\":\"Business\",\"itunes_podcast_secondary_subcategory\":\"\",\"itunes_email\":null,\"language\":null,\"itunes_explicit\":false,\"podcast_art_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/fda09ed9-d5f7-42e0-98f7-e60729d6e96e_3000x3000.png\",\"podcast_subtitle\":\"\",\"podcast_keywords\":null,\"hide_podcast_feed_link\":false,\"podcast_feed_url\":null,\"podcast_title\":\"Reliability Enablers\",\"podcast_description\":\"Software reliability is a tough topic for engineers in many organizations. The Reliability Enablers (Ash Patel and Sebastian Vietz) know this from experience. Join us as we demystify reliability jargon like SRE, DevOps, and more. We interview experts and share practical insights. Our mission is to help you boost your success in reliability-enabling areas like observability, incident response, release engineering, and more. \",\"podcast_byline\":\"Ash Patel & Sebastian Vietz\",\"paid_podcast_episode_art_url\":null,\"migrated_to_attached_podcast\":false},\"showLinks\":[{\"id\":14907,\"publication_id\":2473001,\"section_id\":120870,\"url\":\"https://podcasts.apple.com/us/podcast/s-r-e-path-podcast/id1683437925\",\"platform\":\"apple_podcasts\"}],\"pageTheme\":null,\"podcastPalette\":{\"Vibrant\":{\"rgb\":[100,4,164],\"population\":608},\"DarkVibrant\":{\"rgb\":[92,4,148],\"population\":312},\"LightVibrant\":{\"rgb\":[201.3285714285713,125.55714285714282,251.84285714285718],\"population\":0},\"Muted\":{\"rgb\":[130,134,116],\"population\":29},\"DarkMuted\":{\"rgb\":[59,76,90],\"population\":139},\"LightMuted\":{\"rgb\":[172,171,172],\"population\":34}},\"spotify_podcast_settings\":{\"id\":7987,\"publication_id\":null,\"section_id\":120870,\"spotify_uri\":\"spotify:show:5J8BtbCX2qRmixOhQFK7O0\",\"spotify_podcast_title\":null,\"created_at\":\"2024-04-22T15:47:09.427Z\",\"updated_at\":\"2024-04-22T15:47:49.120Z\",\"currently_published_on_spotify\":true,\"spotify_show_url\":\"https://open.spotify.com/show/5J8BtbCX2qRmixOhQFK7O0\"},\"unified_podcast_settings\":null},{\"id\":127197,\"created_at\":\"2024-05-24T04:47:56.716Z\",\"updated_at\":\"2024-06-05T08:30:16.840Z\",\"publication_id\":2473001,\"name\":\"Reliability Blueprint\",\"description\":\"I'm gradually developing a framework for driving increased software reliability. Access it via https://www.srepath.com/reliability-blueprint/\",\"slug\":\"software-reliability-framework\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":false,\"sibling_rank\":3,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":true,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":127243,\"created_at\":\"2024-05-24T13:10:28.400Z\",\"updated_at\":\"2024-05-24T13:10:35.930Z\",\"publication_id\":2473001,\"name\":\"Reliability Tools\",\"description\":\"Quick rundown of reliability tools as I get the time to look into them\",\"slug\":\"reliability-tools\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":4,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}],\"podcastTabInfo\":{\"hasMultiplePodcasts\":false,\"firstPodcastSectionId\":120870},\"multipub_migration\":null,\"navigationBarItems\":[{\"id\":\"0216016c-f337-49d8-a556-7a0f3f3845a5\",\"publication_id\":2473001,\"sibling_rank\":0,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":true,\"standard_key\":\"leaderboard\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"postTag\":null,\"section\":null,\"children\":[]},{\"id\":\"0b772df7-61c1-47ad-9354-f1fab5c8c459\",\"publication_id\":2473001,\"sibling_rank\":9999,\"link_title\":null,\"link_url\":null,\"section_id\":116798,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"postTag\":null,\"section\":{\"id\":116798,\"slug\":\"observability\",\"name\":\"Observability focus\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/b2430b5d-ad76-4ca7-99d3-ba22adc5bd1a_1280x1280.png\",\"newsletter_enabled\":null},\"children\":[]},{\"id\":\"a4a91532-783f-4935-a604-93c8e9e40995\",\"publication_id\":2473001,\"sibling_rank\":9999,\"link_title\":null,\"link_url\":null,\"section_id\":127197,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"postTag\":null,\"section\":{\"id\":127197,\"slug\":\"software-reliability-framework\",\"name\":\"Reliability Blueprint\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":null,\"newsletter_enabled\":null},\"children\":[]},{\"id\":\"5be6de6d-e5d1-46b5-bb92-66370aea74a6\",\"publication_id\":2473001,\"sibling_rank\":9999,\"link_title\":null,\"link_url\":null,\"section_id\":127243,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"postTag\":null,\"section\":{\"id\":127243,\"slug\":\"reliability-tools\",\"name\":\"Reliability Tools\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":null,\"newsletter_enabled\":null},\"children\":[]},{\"id\":\"46fd031d-c951-4703-ae4b-d26681b366f5\",\"publication_id\":2473001,\"sibling_rank\":9999,\"link_title\":null,\"link_url\":null,\"section_id\":120870,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"postTag\":null,\"section\":{\"id\":120870,\"slug\":\"podcast\",\"name\":\"Hear the Podcast\",\"hide_from_navbar\":false,\"is_podcast\":true,\"logo_url\":null,\"newsletter_enabled\":null},\"children\":[]}],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Ash Patel\",\"handle\":\"srepath\",\"role\":\"admin\",\"owner\":true,\"user_id\":19216588,\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg\",\"bio\":\"Independent reliability advocate working to drive greater software reliability beyond BigTech. Previously a seasoned operations leader.\",\"status\":null},{\"name\":\"Sebastian Vietz\",\"handle\":\"sreinsights\",\"role\":\"admin\",\"owner\":false,\"user_id\":103627353,\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/72b7b9d5-d70a-4293-9a4d-f3b0b964afd1_1550x1550.jpeg\",\"bio\":\"Reliability Engineering Advocate, Enabler of People and Chief Naming Officer ;-)\",\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"threads_v2_enabled\":false,\"viralGiftsConfig\":null,\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://read.srepath.com\",\"hostname\":\"read.srepath.com\",\"is_on_substack\":false,\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"pageThemes\":{\"podcast\":null},\"multiple_pins\":true,\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#121bfa\",\"primary\":{\"r\":18,\"g\":27,\"b\":250,\"a\":1},\"primary_hover\":{\"r\":0,\"g\":64,\"b\":165,\"a\":1},\"primary_elevated\":{\"r\":0,\"g\":64,\"b\":165,\"a\":1},\"secondary\":{\"r\":18,\"g\":27,\"b\":250,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":18,\"g\":27,\"b\":250,\"a\":0.2},\"bg_hover\":{\"r\":18,\"g\":27,\"b\":250,\"a\":0.3},\"dark\":{\"primary\":{\"r\":18,\"g\":27,\"b\":250,\"a\":1},\"primary_hover\":{\"r\":65,\"g\":51,\"b\":255,\"a\":1},\"primary_elevated\":{\"r\":65,\"g\":51,\"b\":255,\"a\":1},\"secondary\":{\"r\":18,\"g\":27,\"b\":250,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":18,\"g\":27,\"b\":250,\"a\":0.2},\"bg_hover\":{\"r\":18,\"g\":27,\"b\":250,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":18,\"g\":27,\"b\":250,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":120,\"g\":111,\"b\":255,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":206.90096477355226,\"g\":206.90096477355175,\"b\":206.9009647735519,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#121BFA\",\"primary\":{\"r\":18,\"g\":27,\"b\":250,\"a\":1},\"primary_hover\":{\"r\":5,\"g\":14,\"b\":237,\"a\":1},\"primary_elevated\":{\"r\":18,\"g\":27,\"b\":250,\"a\":1},\"secondary\":{\"r\":18,\"g\":27,\"b\":250,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":18,\"g\":27,\"b\":250,\"a\":1},\"primary_hover\":{\"r\":65,\"g\":51,\"b\":255,\"a\":1},\"primary_elevated\":{\"r\":65,\"g\":51,\"b\":255,\"a\":1},\"secondary\":{\"r\":18,\"g\":27,\"b\":250,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":18,\"g\":27,\"b\":250,\"a\":0.2},\"bg_hover\":{\"r\":18,\"g\":27,\"b\":250,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"secondary\":{\"r\":134,\"g\":135,\"b\":135,\"a\":1},\"tertiary\":{\"r\":146,\"g\":146,\"b\":146,\"a\":1},\"accent\":{\"r\":18,\"g\":27,\"b\":250,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":120,\"g\":111,\"b\":255,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"secondary\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"secondary_elevated\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"tertiary\":{\"r\":221,\"g\":221,\"b\":221,\"a\":1},\"quaternary\":{\"r\":183,\"g\":183,\"b\":183,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}},\"wordmark_bg\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"is_dark\":false}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[28,140,164],\"population\":375},\"DarkVibrant\":{\"rgb\":[20,140,163],\"population\":105},\"LightVibrant\":{\"rgb\":[141.73749999999998,219.0875,235.6625],\"population\":0},\"Muted\":{\"rgb\":[100,128,140],\"population\":8},\"DarkMuted\":{\"rgb\":[61,96,107],\"population\":316},\"LightMuted\":{\"rgb\":[181,195,201],\"population\":49}},\"section\":{\"id\":116798,\"created_at\":\"2024-04-01T07:43:46.582Z\",\"updated_at\":\"2024-04-01T07:52:50.349Z\",\"publication_id\":2473001,\"name\":\"Observability focus\",\"description\":\"Covering concepts related to observability (aka o11y)\",\"slug\":\"observability\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":1,\"port_status\":\"success\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/b2430b5d-ad76-4ca7-99d3-ba22adc5bd1a_1280x1280.png\",\"hide_from_navbar\":false,\"email_from_name\":\"Ash Patel\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":false,\"customDomain\":\"read.srepath.com\"},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":false,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":false,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":null,\"twitterPixelId\":null,\"parselyPixelId\":null,\"googleTagManagerToken\":null,\"googleAnalytics4Token\":null,\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":false,\"credit_token_enabled\":false,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":false,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":false,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":null,\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"\",\"notes_feed_enabled\":false,\"paywall_unlock_tokens\":false,\"post_preview_crop_gravity\":\"auto\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":true,\"reader_referrals_leaderboard_enabled\":true,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":false,\"local_currency_modal_seen\":false,\"local_payment_methods_modal_seen\":false,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":false,\"welcome_page_opt_out_text\":\"No thanks\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":false,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://read.srepath.com/p/observability-cardinality-conundrum\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":143098468,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2024-04-09T12:00:33.638Z\",\"updated_at\":\"2024-04-09T12:00:33.841Z\",\"publication_id\":2473001,\"search_engine_description\":null,\"search_engine_title\":null,\"section_id\":116798,\"should_send_free_preview\":false,\"show_guest_bios\":true,\"slug\":\"observability-cardinality-conundrum\",\"social_title\":null,\"subtitle\":\"\",\"teaser_post_eligible\":true,\"title\":\"Solving Observability's Cardinality Conundrum\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"everyone\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":0,\"reactions\":{\"\u2764\":12},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"boost-observability-usability\",\"next_post_slug\":\"observability-key-concepts\",\"cover_image\":\"https://substack-post-media.s3.amazonaws.com/public/images/fd726afa-fea9-4584-8816-54ee574df401_1024x540.png\",\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":{\"post_id\":143098468,\"podcast_episode_number\":null,\"podcast_season_number\":null,\"podcast_episode_type\":null,\"should_syndicate_to_other_feed\":null,\"syndicate_to_section_id\":null,\"hide_from_feed\":false,\"free_podcast_url\":null,\"free_podcast_duration\":null,\"preview_contains_ad\":false,\"was_imported_self_serve_sync\":false,\"draft_free_podcast_url\":null,\"draft_free_podcast_duration\":null},\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"Introduction\",\"body_html\":\"<h2>Introduction</h2><p>Cardinality is a term you\u2019ll hear over and over again if you\u2019re looking into how to do observability.</p><p>And especially if you are talking with vendors! They love this topic!</p><p>A lot of people have been thinking about high cardinality for a while and for a good reason.</p><p>Because high cardinality can cost you a LOT of money and time if you go about it wrong\u2026 but really, what we want to do is <strong>cut down</strong> <em><strong>excessive</strong></em><strong> cardinality</strong>.</p><p>The kind that doesn\u2019t add value to your querying and intelligence.</p><p>That\u2019s why dealing with cardinality is not a straightforward solution. Cutting labels willy-nilly is not the answer. We\u2019ll get onto ways to deal with this later on.</p><p>But we should first talk about what cardinality actually means.</p><h2>What is cardinality?</h2><p>Because it\u2019s not something you think about every day\u2026 unless you\u2019re an observability engineer or vendor.</p><p>The last time I heard \u201Ccardinality\u201D being used this often was in my SQL classes, which was a while back.</p><p>So I did a refresher and went down a rabbit hole of math, logic, and all that fun stuff.</p><p>Cardinality refers to <em><strong>how many unique values there are in a data set.</strong></em></p><p>You\u2019re essentially looking for how diverse \u2014 and ultimately complex \u2014 the data is.</p><h3>How do you differentiate between low and high cardinality data?</h3><ul><li><p>Low cardinality means low complexity with few dimensions to data. This is fine for analyzing aggregate data but lacks granularity that engineers often seek to solve system problems</p></li><li><p>High cardinality contains more data dimensions. This lets you slice and dice data for more detailed analysis, but you then have to deal with complexity issues.</p></li></ul><p>By dimensions, I mean attributes like method, status_code, instance_id, etc.</p><p><strong>Let\u2019s run through a simple example to cement the difference:</strong></p><p>Let\u2019s say you have the table <em>fruits</em> with only the fruit <em>apple</em> in your database. We want to add a <em>color</em> key to identify each apple in your table.</p><p>Now, the data on apples is low cardinality if you only find red or green apples like so:</p><p>But it becomes higher cardinality if you have apples with colors like orange, blue, red, pink, green, violet, black etc.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!ZNbn!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!ZNbn!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 424w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 848w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 1272w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!ZNbn!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":null,"width":null,"resizeWidth":null,"bytes":null,"alt":"Low vs high cardinality data example visual","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"Low vs high cardinality data example visual\\\" title=\\\"Low vs high cardinality data example visual\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!ZNbn!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 424w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 848w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 1272w, https://substackcdn.com/image/fetch/$s_!ZNbn!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92eb2ed1-2439-4109-a02e-c9360acd104e_1024x540.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>The more color options you have for <em>apple</em>, the higher the cardinality of the <em>color</em> data.</p><h3>Examples of high cardinality</h3><p>Here are some examples of high cardinality data:</p><ul><li><p>email addresses (never append these to metrics!)</p></li><li><p>user IDs (<em>\u201DObservability engineer: why is this in my beautiful TSDB?!\u201D</em>)</p></li><li><p>IP addresses (sometimes appended for AppSec purposes)</p></li><li><p>instance name (fair use case for identifying instance issues)</p></li><li><p>pod s (like the kind you find within Kubernetes)</p></li></ul><h2>3 key benefits of high cardinality in observability data</h2><p>High cardinality enables these traits within observability data:</p><h3>1. Granularity</h3><p>With granular data, you can slice and dice to deeper and deeper levels to precisely pinpoint where issues like performance degradation and outages are happening.</p><p>For example, with each <code>500 error</code>, you may want to dig a little further.</p><p>With a high cardinality metric, you can dig into the 50x for <code>GET</code>, <code>POST</code>, and <code>DELETE</code> requests.</p><h3>2. Segmentation</h3><p>Helps you collect disparate data and organize it into chunks for easier human processing.</p><p>Examples include:</p><ul><li><p>payments by age group e.g. 18-25, 25-34, 35-49, etc.</p></li><li><p>payments by market region e.g. North America, South East Asia, LatAm, EU, etc.</p></li></ul><h3>3. Performance evaluation</h3><p>High cardinality means you can go beyond \u201Csuccess\u201D and \u201Cfail\u201D for responses.</p><p>You can classify it as performance levels to improve reliability for different instances and groups.</p><p>Imagine what you can work out by putting responses into buckets of 0-10 scoring.</p><p><strong>Before we continue, I will assume that you know the answers to the following questions:</strong></p><p>What does observability data look like?</p><p>How does observability data flow?</p><p>Where observability data gets stored and how</p><h2>How do observability data types do in terms of cardinality?</h2><p>Metrics are the main data type when we think about cardinality. But let\u2019s still cover each of the 3 main observability data types to see its cardinality issue:</p><h3>High cardinality in l<strong>ogs</strong></h3><p>Logs of yesteryear are typically less affected by high cardinality than metrics and traces.</p><p>Some engineers still use unstructured logs for small-scale systems.</p><p>At the commercial scale, modern logs are structured or at least shifting toward that. JSON shows the most promise through that.</p><p>So why shift to a structured format? A few things come into play like:</p><ul><li><p>Readability</p></li><li><p>Better ability to group or segment logs by keys for better more refined analytical outcomes</p></li><li><p>Speed to read as less string manipulation or string processing has to take place during the reading of the aggregated log data</p></li></ul><p>You can derive metrics from logs now!</p><p>Logs in a commercial setting can experience high cardinality, just like metrics and traces.</p><h3>High cardinality in m<strong>etrics</strong></h3><p>Among the trio, metrics are the most significantly affected by high cardinality.</p><p>Metrics get complicated by the fact that they can have many dimensions.</p><p>You can have a multiplier effect when you add a new dimension.</p><p>This multiplication is what defines the high cardinality we are talking about.</p><p>You benefit in one way from this through more granularity that helps deeper analysis.</p><p>But then you also impact storage, querying, and visualization performance.</p><h3>High cardinality in t<strong>races</strong></h3><p>Traces can also be affected by high cardinality.</p><p>Picture a trace to be the bus route from New York to Los Angeles and back.</p><p>This route can be divided into sections \u2014 when you plan a trip along this route you may want to stop for a breather, food, or a bathroom break.</p><p>Each section of this route or round trip is equivalent to what in tracing we would call a span.</p><p>A common dimension for spans is the time or duration of that specific section of work.</p><p>We can add more detail or context for each span by appending metadata to the span.</p><p>You get high cardinality within tracing when you use spans like user IDs and specific names of what happened.</p><p>All this once again impacts storage and querying performance.</p><h2>Calculating the impact of cardinality within a metric</h2><p>Adding a dimension to a metric does not cause high cardinality.</p><p>It\u2019s what that key-value pair stands for that determines this.</p><p>So adding a 6th dimension doesn\u2019t increase cardinality if it\u2019s a boolean or a \u201Csuccess\u201D/\u201Dfail\u201D type.</p><p>But if it were something like instance ID and you have 100 or 1000 instances, that would.</p><p>We do not need to calculate cardinality unless we are sitting in a math class.</p><p>We are more interested in how many time series</p><h3>Calculating time series count for a metric</h3><p>Let\u2019s run through a simple example.</p><p>Say you have a metric called <em>network_latency_distribution</em> covering 100 instances with 10 buckets, 10 possible response codes, and 10 network paths.</p><pre><code>**Calculating the series would look like this:**\\n= 100 instances * 10 buckets * 10 response codes * 10 paths\\n= 100,000 series\\n</code></pre><p>This is a reasonable size for a series, but things can get out of hand as you add more dimensions.</p><p><strong>Adding a dimension like </strong><em><strong>region</strong></em><strong> can significantly increase cardinality.</strong></p><pre><code>**Say we have 6 regions to choose from. The calculation would become:**\\n= 100 instances x 10 buckets x 10 response codes x 10 paths x 6 regions\\n= 600,000 series (! \uD83D\uDE15)\\n</code></pre><p>Okay. It doesn\u2019t look excessive, but can significantly increase querying time!</p><p>I\u2019ll share some query time data with you in a minute.</p><p><strong>The dimension of </strong><em><strong>pods</strong></em><strong> (within containers) can increase cardinality by a huge factor, too.</strong></p><pre><code>**Say we have a low 1,000 pods in action. The calculation would become:**\\n= 100 instances x 10 buckets x 10 response codes x 10 paths x 1,000 pods\\n= 100,000,000 series (!! \uD83D\uDE26)\\n</code></pre><p><strong>Excessive cardinality comes into the picture when you add a dimension like user_id.</strong></p><pre><code>**Even with a modest user count of 10,000, the time series blows out with:**\\n= 100 instances * 10 buckets * 10 response codes * 10 paths * 10000 users\\n= 1,000,000,000 series (!!! \uD83D\uDE31)\\n</code></pre><p>We\u2019ll talk about Excessive cardinality in more detail later on.</p><p>\uD83D\uDEA8 <strong>PSA:</strong> The more types of data you collect, the more fragmented view you have of the whole piece. And if you have a fragmented view, that reduces your ability to TAKE ACTION on the data. Remember that <strong>Quality of data > Quantity of data</strong>.</p><h2>High cardinality data is rising because of system trends</h2><h3>Shift from monolith to microservices architecture</h3><p>Where you had one humongous service emitting metrics, you now have 10, 20, 50, 100+ microservices with each emitting its metrics. \u2018Nuf said.</p><h3>Shift from VM-based to container-based infrastructure</h3><p>Who would\u2019ve thought that life was easier when we depended on VM infrastructure rather than containers? </sarcasm></p><p>Containers have their benefits but generate a whole bunch of time-series data.</p><p>For every 1 VM in an older system, there are 10s of containers to match workloads.</p><p>The key culprit behind this is their <em><strong>high ephemerality</strong></em>.</p><p>The ephemeral nature of containers refers to when they start, shut down and then a new container takes their place.</p><p>All this stop-start emits a lot of data for metrics collectors to ingest and push to storage.</p><h3>Serverless functions as a large part of the system</h3><p>Every time you invoke a Lambda function, that induces a metric with a time series. Depending on the particular service you are running on serverless, this time series number can be HUGE.</p><p>A whole bunch of factors come into play like:</p><ol><li><p>how often you're invoking the lambda function</p></li><li><p>the number of serverless functions in your system</p></li><li><p>concurrency of handling requests and scale</p></li></ol><p>Your time series data incurs higher and higher cardinality as these 3 factors rise in frequency or occurrence (depending on the factor).</p><h2>When to deal with high cardinality</h2><p>Remember how observability data flows. There are 4 distinct stages:</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!PUeG!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!PUeG!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 424w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 848w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 1272w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!PUeG!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":null,"width":null,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" title=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!PUeG!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 424w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 848w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 1272w, https://substackcdn.com/image/fetch/$s_!PUeG!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F53f01e36-55fa-4fae-ad38-5e8caac71fa0_1024x288.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>Dealing with high cardinality at the <em>Instrumentation</em> stage is not ideal. You will not confidently know what dimensions to filter out at this stage.</p><p>It\u2019s also not ideal at the <em>Usage</em> stage. It\u2019s too late to deal with cardinality issues because you\u2019ve incurred high ingestion and storage costs, so\u2026</p><p>Deal with high cardinality at the <em>Ingestion & Storage</em> stages. This is when you can put in practices like in-flight aggregation, cardinality isolation, and cardinality limiters.</p><h2>When to ditch high cardinality data</h2><p>High cardinality data is a necessary evil in some situations. As I mentioned earlier, it helps with granularity, segmentation, and deeper performance evaluation.</p><p>If it does not add value to your querying, intelligence, or alerting. Ditch it.</p><p>It\u2019s just an ornament to give you pretty data.</p><p>What I mean by this is it <em>looks</em> like it <em>could</em> be important.</p><p>But in reality:</p><ul><li><p>it\u2019s giving you limited analytical value</p></li><li><p>rarely contributes to usable insight <strong>and</strong></p></li><li><p>with either of the above two is resource intensive.</p></li></ul><p>Because if resource intensiveness did not cost money or time, we wouldn\u2019t care about it.</p><p>We\u2019d just let high cardinality data sit there and do its thing.</p><p>But it does cost money in terms of storage and processing power.</p><p>And it costs time in terms of how long you have to wait before you can start working with the data.</p><p>And it\u2019s definitively something you need to look into if it\u2019s slowing things down to the point where you\u2019re pushing past your MTTR target.</p><p>(MTTR = your mean time to recovery, repair, resolution, whatever you want to call it).</p><p>Now, you might be saying thank you, Captain Obvious.</p><p>But think about how often this is still a real problem in software systems.</p><p>You need to pose it as a challenge for people to think about and solve it.</p><h2>The problem with <em>excessive</em> cardinality</h2><p>The working group behind the open-source Prometheus monitoring tool has warned about this for a while.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!PQcc!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!PQcc!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 424w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 848w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 1272w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!PQcc!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":null,"width":null,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" title=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!PQcc!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 424w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 848w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 1272w, https://substackcdn.com/image/fetch/$s_!PQcc!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2b851b84-dee2-4011-8c7c-9218741d74f7_1024x173.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>Excerpt from <a href=\\\"http://Prometheus.io\\\">Prometheus.io</a> documentation on <a href=\\\"https://prometheus.io/docs/practices/naming/\\\">Metric and label naming</a></p><p>A quick TLDR of their alert message on key-value pairs:</p><blockquote><p>\u201CEvery unique combination of key-value pairs represents a new time-series. This significantly increases the data stored.\u201D</p></blockquote><p>An ever-increasing number of possible key-value pairs or dimensions does something sinister within time series databases (TSDBs).</p><p>The number of series for a single metric will explode \uD83D\uDCA5 to the point that your querying eventually slows down to a crash.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!0EGq!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!0EGq!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 424w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 848w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 1272w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!0EGq!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":null,"width":null,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" title=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!0EGq!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 424w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 848w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 1272w, https://substackcdn.com/image/fetch/$s_!0EGq!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0698506b-f6d3-4d8c-9be5-fcdee4925179_636x615.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>Query performance data by Chris Marchbanks (ex-Splunk, Grafana Labs) says it all.</p><p>This is how long it took to query quantities of time series:</p><ul><li><p>100,000 series took 1.5 seconds (acceptable)</p></li><li><p>200,000 series took 5 seconds (slow-ish)</p></li><li><p>10,000,000 series took 15 minutes (!)</p></li></ul><p>Keep this in mind: high or (better put) <strong>excessive cardinality is a data problem at its core</strong>.</p><p>Not only does it slow down query time to a grind, but it can also cause trouble like:</p><ul><li><p>data being wasted on dimensions that are not needed for system improvements</p></li><li><p>need for engineers to constantly maintain performance of the observability system</p></li><li><p>higher storage needs and processing resources means higher cost to run</p></li></ul><p>We will think through a few solution starters toward the end to prevent or at least reduce this risk.</p><h2>What\u2019s contributing to excessive cardinality in a software system?</h2><h3>Bad dimension selection</h3><p>Remember my calculation example above?</p><p>Putting in user or request IDs can skyrocket your observability metrics\u2019 cardinality.</p><p>Each unique identifier will contribute to a new entry, increasing the overall cardinality of the logs.</p><h3>Improper sample practices</h3><p>Observability systems generate a ton of data all the time.</p><p>It\u2019s not an easy task to query all of that data all the time.</p><p>This is where you want to bring sampling practices to select a portion of the data to analyze.</p><p>But if you don\u2019t use the right sampling techniques, you will deal with high cardinality data and more series than your dashboard or query tool can handle.</p><h3>Unbounded event types</h3><p>Metrics are not the sole culprits for pushing out excessive cardinality data.</p><p>Putting weak boundaries around your event data can do the same to logs and traces.</p><p>A good system will have few event types while a system suffering from excessive cardinality will have numerous possible event types.</p><p>The more event types there are, the more logs and spans you\u2019ll have to push to ingestion.</p><p>This is not an exhaustive list of ways excessive cardinality can happen. I want to illustrate the idea that there are several ways you can end up with it.</p><h2>How to solve <em>excessive</em> cardinality</h2><h3>Be selective about dimensions</h3><p>I feel like I\u2019ve mentioned it several times already, but never, ever use dimensions like email address, user ID, transaction ID, or anything with overly unique data in your metrics.</p><h3>Split the metric into smaller metrics</h3><p>Ask yourself these two questions:</p><blockquote><p>Do I need to have this single metric with all these dimensions?</p></blockquote><blockquote><p>Can I split it into two separate metrics that can still help me answer the questions I will pose at querying, and still give me the necessary alerting?</p></blockquote><p>An example might better highlight why you\u2019d want to do this:</p><p>Let\u2019s return to our metric called <em>network_latency_distribution, which</em> covers 100 instances with 10 buckets, 10 possible response codes, and 10 network paths.</p><pre><code>**Calculating the series would look like this:**\\n= 100 instances * 10 buckets * 10 response codes * 10 paths\\n= 100,000 series\\n</code></pre><p>Now what would happen if we were to split this metric into 2 individual metrics, one without paths and one without buckets?</p><pre><code>**Here's the first metric without paths:**\\n= 100 instances * 10 buckets * 10 response codes \\n= 10,000 series\\n</code></pre><pre><code>**Here's the second metric without buckets:**\\n= 100 instances * 10 response codes * 10 paths\\n= 10,000 series\\n</code></pre><p>This gives us a grand total of 20,000 series and <strong>a whopping 80,000 series reduction</strong>!</p><p>This split works perfectly if we don\u2019t need to correlate paths with buckets to solve system issues.</p><p>How much faster would it be to query 20,000 vs 100,000 series?</p><p>Answer: enough to feel instantaneous vs. time spent hearing, \u201CPlease wait while we process your query.\u201D</p><h3>Allow high(er) cardinality for high-value metrics only</h3><p>It can still make sense to generate a whole bunch of series for metrics that add business value.</p><p>You may have some metrics that need 100,000 series rather than being split into multiple mini-metrics and losing their strength.</p><p>How do we define a high-value metric? It is this if it supports:</p><ul><li><p>critical decision-making processes</p></li><li><p>developing actionable insights or</p></li><li><p>enhancement of overall system performance</p></li></ul><p>What hits one of these criteria depends on your software architecture, industry context, and the problems you\u2019re looking to solve.</p><p>Here are a few considerations to make:</p><ol><li><p>These kinds of metrics might not be suitable for time-of-essence needs like alerting</p></li><li><p>You can try and save the designation of high-value metrics to ad-hoc queries, low-usage dashboards for specialist or special interest groups, and periodic reports</p></li><li><p>You still need to weigh the impact vs querying & visualization time</p></li><li><p>Even a high-value metric can start looking too expensive with the way observability is priced these days, so work out the costings vs the value you attain and discuss with management</p></li></ol><p>We must still remember to control cardinality levels to meet our cost and time-to-productivity constraints.</p><h2>Wrapping up</h2><p>If you can only remember one thing from this guide, I want it to be this \uD83D\uDC47\uD83C\uDFFC</p><p><strong>Cardinality is valuable, but excessive cardinality is expensive \u2014 in terms of time to query, cost to store, and resource consumption to process & analyze. Keep it in check.</strong></p>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"Introduction\",\"wordcount\":2854,\"post_preview_limit\":null,\"language\":\"en\",\"postTags\":[{\"id\":\"3e148911-199d-444c-a98f-5a1cffedf904\",\"publication_id\":2473001,\"name\":\"observability\",\"slug\":\"observability\",\"hidden\":false}],\"postCountryBlocks\":[],\"headlineTest\":null,\"coverImagePalette\":{\"Vibrant\":{\"rgb\":[206,67,91],\"population\":62},\"DarkVibrant\":{\"rgb\":[68,81,14],\"population\":333},\"LightVibrant\":{\"rgb\":[208,212,165],\"population\":53},\"Muted\":{\"rgb\":[146,124,85],\"population\":390},\"DarkMuted\":{\"rgb\":[42,64,72],\"population\":129},\"LightMuted\":{\"rgb\":[204,165,167],\"population\":32}},\"publishedBylines\":[{\"id\":19216588,\"name\":\"Ash Patel\",\"handle\":\"srepath\",\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/4d75e9bc-eb68-40e8-80ca-40ae78e2d204_2048x2560.jpeg\",\"bio\":\"Independent reliability advocate working to drive greater software reliability beyond BigTech. Previously a seasoned operations leader.\",\"profile_set_up_at\":\"2024-03-30T10:17:13.605Z\",\"reader_installed_at\":null,\"publicationUsers\":[{\"id\":2501537,\"user_id\":19216588,\"publication_id\":2473001,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":2473001,\"name\":\"Reliability Enablers\",\"subdomain\":\"srepath\",\"custom_domain\":\"read.srepath.com\",\"custom_domain_optional\":false,\"hero_text\":\"Soon, focus will shift to the emerging intersection of resilient and intelligent systems. Because \u201Cfive nines\u201D won\u2019t survive AI ignorance. Newsletter and podcast inside.\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/99ee1dc2-77bf-4ffa-b056-f66dac8ad0d0_128x128.png\",\"author_id\":19216588,\"primary_user_id\":19216588,\"theme_var_background_pop\":\"#121BFA\",\"created_at\":\"2024-03-30T10:17:21.113Z\",\"email_from_name\":\"Ash Patel\",\"copyright\":\"Ash P\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"magaziney\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"is_guest\":false,\"bestseller_tier\":null,\"status\":null}],\"reaction\":null,\"reaction_count\":12,\"comment_count\":2,\"child_comment_count\":1,\"is_geoblocked\":false,\"hasCashtag\":false,\"section\":{\"id\":116798,\"created_at\":\"2024-04-01T07:43:46.582Z\",\"updated_at\":\"2024-04-01T07:52:50.349Z\",\"publication_id\":2473001,\"name\":\"Observability focus\",\"description\":\"Covering concepts related to observability (aka o11y)\",\"slug\":\"observability\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":1,\"port_status\":\"success\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/b2430b5d-ad76-4ca7-99d3-ba22adc5bd1a_1280x1280.png\",\"hide_from_navbar\":false,\"email_from_name\":\"Ash Patel\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}},\"comments\":[{\"id\":55063338,\"body\":\"Great post!! Thanks.\",\"body_json\":{\"type\":\"doc\",\"attrs\":{\"schemaVersion\":\"v1\"},\"content\":[{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"Great post!! Thanks.\"}]}]},\"publication_id\":2473001,\"post_id\":143098468,\"user_id\":16078870,\"ancestor_path\":\"\",\"type\":\"comment\",\"deleted\":false,\"date\":\"2024-04-29T03:57:42.765Z\",\"edited_at\":null,\"status\":\"published\",\"pinned_by_user_id\":null,\"restacks\":0,\"name\":\"Mark Seery\",\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/a81e1337-ef00-4d85-9248-5707e7494ad5_800x800.png\",\"handle\":\"techdynamics\",\"reactor_names\":[\"Ash Patel\"],\"reaction\":null,\"reactions\":{\"\u2764\":1},\"reaction_count\":1,\"children\":[],\"bans\":[],\"suppressed\":false,\"user_banned\":false,\"user_banned_for_comment\":false,\"user_slug\":\"techdynamics\",\"metadata\":{\"is_author\":false,\"membership_state\":null,\"eligibleForGift\":false,\"author_on_other_pub\":{\"name\":\"Tech Dynamics\",\"id\":94215,\"base_url\":\"https://techdynamics.substack.com\"}},\"user_bestseller_tier\":null,\"can_dm\":true,\"userStatus\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null},\"score\":6,\"children_count\":1,\"reported_by_user\":false,\"restacked\":false,\"childrenSummary\":\"1 reply by Ash Patel\"}],\"canonicalUrl\":\"https://read.srepath.com/p/observability-cardinality-conundrum\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://read.srepath.com/p/observability-cardinality-conundrum\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":\"#121BFA\",\"background_pop\":\"#121BFA\",\"color_theme_bg_web\":\"#ffffff\",\"cover_bg_color\":\"#ffffff\",\"cover_bg_color_secondary\":\"#f0f0f0\",\"background_pop_darken\":\"#050eed\",\"print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_darken\":\"#050eed\",\"color_theme_print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_20\":\"rgba(18, 27, 250, 0.2)\",\"color_theme_bg_pop_30\":\"rgba(18, 27, 250, 0.3)\",\"print_pop\":\"#121bfa\",\"color_theme_accent\":\"#121bfa\",\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#121bfa\",\"home_hero\":\"magaziney\",\"home_posts\":\"list\",\"web_bg_color\":\"#ffffff\",\"background_contrast_1\":\"#f0f0f0\",\"background_contrast_2\":\"#dddddd\",\"background_contrast_3\":\"#b7b7b7\",\"background_contrast_4\":\"#929292\",\"background_contrast_5\":\"#515151\",\"color_theme_bg_contrast_1\":\"#f0f0f0\",\"color_theme_bg_contrast_2\":\"#dddddd\",\"color_theme_bg_contrast_3\":\"#b7b7b7\",\"color_theme_bg_contrast_4\":\"#929292\",\"color_theme_bg_contrast_5\":\"#515151\",\"color_theme_bg_elevated\":\"#ffffff\",\"color_theme_bg_elevated_secondary\":\"#f0f0f0\",\"color_theme_bg_elevated_tertiary\":\"#dddddd\",\"color_theme_detail\":\"#e6e6e6\",\"background_contrast_pop\":\"rgba(18, 27, 250, 0.4)\",\"color_theme_bg_contrast_pop\":\"rgba(18, 27, 250, 0.4)\",\"theme_bg_is_dark\":\"0\",\"print_on_web_bg_color\":\"#363737\",\"print_secondary_on_web_bg_color\":\"#868787\",\"background_pop_rgb\":\"18, 27, 250\",\"color_theme_bg_pop_rgb\":\"18, 27, 250\",\"color_theme_accent_rgb\":\"18, 27, 250\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":null,\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"125.0\",\"major\":\"125\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
|
||
<script nonce="igm3B5YFT1CqZB3oVHDEQw">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"srepath\",\"publication_id\":2473001,\"has_plans\":false,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"section_name\":\"Observability focus\",\"section_slug\":\"observability\",\"section_id\":116798,\"section_is_podcast\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
|
||
|
||
|
||
|
||
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.7371871e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.571db0c6.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.a0731d37.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.d196682b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.2686bb5d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.ae8857e4.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.4614ce58.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.41564edd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.b2140fcd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.f3dc2b43.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.7aa24e85.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.1820773d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.765aca5a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.2f1bdde5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.4e726ad5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.f03f24de.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.f579d62b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.80c75c78.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.92b9fc9c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.258e49f2.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.d493457b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.ce61e623.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.015b0f71.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.f1207089.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.b91ab2ef.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.b93f6845.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89356.66161d1e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.211a7337.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47925.24a01d36.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.03bd4c41.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71505.3dc43e33.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.5d148f2c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.86d95e8c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.a5a682c3.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/86118.ff1ec843.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.7fe128ac.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.c78aabc1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.d6e9f8ec.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/29231.a7af5911.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.2b32bbb6.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.f1c17ded.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.1d0af2b9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.951c11cb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.58e078b0.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.dfde36c8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.94c6b3f1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.a327e9f4.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.8bb0d59c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.8f5c5991.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.cb326b80.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.680833b5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.df6a856d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.cf3bcc2c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.8a1bc5dd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.3f20fb36.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.bdbc4e21.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58483.3adddb8a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.0a97506f.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.09a9082f.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.7f0face7.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/92610.a4fc1daf.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.cb10b4b8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11244.b723f409.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99260.28b0ed6a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.7d5c5ba3.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/76768.9a2f2bae.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/41932.6c688f22.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12114.979fb38e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.9732c6f9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.4f5ae3be.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53871.2923c5c9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.588a06d1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99068.bd2e1fba.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.79d3f415.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.c7f428fb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.098bcbcf.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.2eb51db5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.72d1d844.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.bbf1c283.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.2e8ed950.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/32552.889dd584.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.c287f2d8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.6c265246.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.9b9761ae.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.b40bf629.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.07cd629b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.35b48b09.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.f37fb930.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.24075434.js" charset="utf-8"></script>
|
||
|
||
|
||
<script nonce="igm3B5YFT1CqZB3oVHDEQw" nomodule>
|
||
(function() {
|
||
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
|
||
var warningDiv = document.createElement('div');
|
||
warningDiv.style.color = 'red';
|
||
warningDiv.style.padding = '10px';
|
||
warningDiv.style.margin = '10px 0';
|
||
warningDiv.style.border = '1px solid red';
|
||
warningDiv.style.backgroundColor = 'lightyellow';
|
||
warningDiv.innerText = message;
|
||
document.body.prepend(warningDiv);
|
||
})();
|
||
</script>
|
||
|
||
|
||
<!-- Datadog Analytics -->
|
||
<script nonce="igm3B5YFT1CqZB3oVHDEQw">
|
||
(function(h,o,u,n,d) {
|
||
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
|
||
d=o.createElement(u);d.async=1;d.src=n
|
||
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
|
||
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
|
||
window.DD_RUM.onReady(function() {
|
||
window.DD_RUM.init({
|
||
clientToken: 'puba71073f072643721169b68f352438710',
|
||
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
|
||
site: 'datadoghq.com',
|
||
service: 'web',
|
||
env: window._preloads.dd_env,
|
||
version: '8b4c00716b2bc0f7ca1fded553014f7e37b5b07a',
|
||
sessionSampleRate: 1,
|
||
sessionReplaySampleRate: 10,
|
||
trackUserInteractions: window._preloads.dd_ti,
|
||
trackResources: true,
|
||
trackLongTasks: true,
|
||
defaultPrivacyLevel: 'mask-user-input',
|
||
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
|
||
});
|
||
})
|
||
</script>
|
||
<!-- End Datadog Analytics -->
|
||
|
||
<!-- Cloudflare Web Analytics -->
|
||
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
|
||
<!-- End Cloudflare Web Analytics -->
|
||
|
||
|
||
<!-- Fallback tracking pixels -->
|
||
|
||
|
||
|
||
|
||
<noscript>
|
||
<style>
|
||
#nojs-banner {
|
||
position: fixed;
|
||
bottom: 0;
|
||
left: 0;
|
||
padding: 16px 16px 16px 32px;
|
||
width: 100%;
|
||
box-sizing: border-box;
|
||
background: red;
|
||
color: white;
|
||
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
|
||
font-size: 13px;
|
||
line-height: 13px;
|
||
}
|
||
#nojs-banner a {
|
||
color: inherit;
|
||
text-decoration: underline;
|
||
}
|
||
</style>
|
||
|
||
<div id="nojs-banner">
|
||
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
|
||
</div>
|
||
</noscript>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="igm3B5YFT1CqZB3oVHDEQw" data-cf-beacon='{"version":"2024.11.0","token":"68cfe66b5c4749e2ba64d4d9640c04c0","spa":2}' crossorigin="anonymous"></script>
|
||
</body>
|
||
</html>
|