Files
nexus/sreweekly/articles/465/02-it-s-a-log-eat-log-world.html
2026-09-12 17:23:01 +08:00

473 lines
250 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8" />
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
<link rel="preconnect" href="https://substackcdn.com" />
<title data-rh="true">It&#x27;s a log eat log world! - by Obakeng Mosadi</title>
<meta data-rh="true" name="theme-color" content="#ffffff"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="It&#x27;s a log eat log world!"/><meta data-rh="true" name="twitter:title" content="It&#x27;s a log eat log world!"/><meta data-rh="true" name="description" content="Let&#x27;s discuss logging - unstructured, structured and canonical log lines - what they are and what value they bring to your production systems."/><meta data-rh="true" property="og:description" content="Let&#x27;s discuss logging - unstructured, structured and canonical log lines - what they are and what value they bring to your production systems."/><meta data-rh="true" name="twitter:description" content="Let&#x27;s discuss logging - unstructured, structured and canonical log lines - what they are and what value they bring to your production systems."/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!Do-H!,w_1200,h_675,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa09059a6-f716-4e46-997e-037d21ff6ace_3636x2415.jpeg"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!gsO0!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fobakeng.substack.com%2Fapi%2Fv1%2Fpost_preview%2F153407591%2Ftwitter.jpg%3Fversion%3D4"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" property="article:modified_time" content="2025-02-08T05:55:11.270Z"/>
<style>
@layer legacy, pencraftReset, pencraft;
</style>
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<link rel="preload" as="font" href="https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhLq3-cXbKD.woff2" crossorigin />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.cee9fbee.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/86118.c25b9f67.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/76768.f15fc605.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/41932.97d2663c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/32552.313415d5.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
<meta name="author" content="Obakeng Mosadi" />
<meta property="og:url" content="https://obakeng.substack.com/p/its-a-log-eat-log-world" />
<link rel="canonical" href="https://obakeng.substack.com/p/its-a-log-eat-log-world" />
<link rel="shortcut icon" href="https://substackcdn.com/icons/substack/favicon.ico" sizes="32x32">
<link rel="icon" href="https://substackcdn.com/icons/substack/icon.svg" type="image/svg+xml">
<link rel="apple-touch-icon" href="https://substackcdn.com/icons/substack/apple-touch-icon.png">
<link rel="alternate" type="application/rss+xml" href="/feed" title="Unreliability."/>
<style>
@font-face{font-family:'Roboto Mono';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0xoDF4xlVMF-BfR8bXMIjhOsXG-q2oeuFoqFrlnANW3CpmeTIwsa4YaDA.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Roboto Mono';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0xoDF4xlVMF-BfR8bXMIjhOsXG-q2oeuFoqFrlnANW-CpmeTIwsa4YaDA.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Roboto Mono';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0xoDF4xlVMF-BfR8bXMIjhOsXG-q2oeuFoqFrlnANW5CpmeTIwsa4YaDA.woff2) format('woff2');unicode-range:U+0370-0377,U+037A-037F,U+0384-038A,U+038C,U+038E-03A1,U+03A3-03FF}@font-face{font-family:'Roboto Mono';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0xoDF4xlVMF-BfR8bXMIjhOsXG-q2oeuFoqFrlnANW1CpmeTIwsa4YaDA.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Roboto Mono';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0xoDF4xlVMF-BfR8bXMIjhOsXG-q2oeuFoqFrlnANW0CpmeTIwsa4YaDA.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Roboto Mono';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0xoDF4xlVMF-BfR8bXMIjhOsXG-q2oeuFoqFrlnANW6CpmeTIwsa4Y.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhGq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhPq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhIq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0370-0377,U+037A-037F,U+0384-038A,U+038C,U+038E-03A1,U+03A3-03FF}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhEq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhFq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhLq3-cXbKD.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhGq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhPq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhIq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0370-0377,U+037A-037F,U+0384-038A,U+038C,U+038E-03A1,U+03A3-03FF}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhEq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhFq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhLq3-cXbKD.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhGq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhPq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhIq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0370-0377,U+037A-037F,U+0384-038A,U+038C,U+038E-03A1,U+03A3-03FF}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhEq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhFq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhLq3-cXbKD.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhGq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhPq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhIq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0370-0377,U+037A-037F,U+0384-038A,U+038C,U+038E-03A1,U+03A3-03FF}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhEq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhFq3-cXbKDO1w.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Roboto Mono';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/robotomono/v31/L0x5DF4xlVMF-BfR8bXMIjhLq3-cXbKD.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
</style>
<style data-pub-theme-prefix>:root{--color_theme_bg_pop:#e11d48;--background_pop:#e11d48;--cover_bg_color:#FFFFFF;--cover_bg_color_secondary:#f0f0f0;--background_pop_darken:#ca1a41;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#ca1a41;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(225, 29, 72, 0.2);--color_theme_bg_pop_30:rgba(225, 29, 72, 0.3);--print_pop:#e11d48;--color_theme_accent:#e11d48;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#e11d48;--font_family_headings_preset:'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';--font_weight_headings_preset:700;--font_family_body_preset:'Roboto Mono',monospace;--font_weight_body_preset:400;--font_size_body_offset:-1px;--font_preset_heading:sans;--font_preset_body:mono;--home_hero:feature-media;--home_posts:list;--home_show_top_posts:true;--web_bg_color:#ffffff;--background_contrast_1:#f0f0f0;--background_contrast_2:#dddddd;--background_contrast_3:#b7b7b7;--background_contrast_4:#929292;--background_contrast_5:#515151;--color_theme_bg_contrast_1:#f0f0f0;--color_theme_bg_contrast_2:#dddddd;--color_theme_bg_contrast_3:#b7b7b7;--color_theme_bg_contrast_4:#929292;--color_theme_bg_contrast_5:#515151;--color_theme_bg_elevated:#ffffff;--color_theme_bg_elevated_secondary:#f0f0f0;--color_theme_bg_elevated_tertiary:#dddddd;--color_theme_detail:#e6e6e6;--background_contrast_pop:rgba(225, 29, 72, 0.4);--color_theme_bg_contrast_pop:rgba(225, 29, 72, 0.4);--theme_bg_is_dark:0;--print_on_web_bg_color:#363737;--print_secondary_on_web_bg_color:#868787;--background_pop_rgb:225, 29, 72;--color_theme_bg_pop_rgb:225, 29, 72;--color_theme_accent_rgb:225, 29, 72;}</style>
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<style data-pub-theme-postfix></style>
</head>
<body class="">
<div id="entry">
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:71px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="flex-basis:0px;flex-grow:1;" class="logoContainer-p12gJb"></div><div style="flex-grow:0;" class="titleContainer-DJYq5v"><h1 class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd"><a href="/" class="pencraft pc-display-contents pc-reset">Unreliability.</a></h1></div><div style="flex-basis:0px;flex-grow:1;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Fits-a-log-eat-log-world&amp;for_pub=obakeng" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="height:72px;"></div></div></div><div><script nonce="XYBzbX0dMnqLivGDUHjKMw" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://obakeng.substack.com/p/its-a-log-eat-log-world","mainEntityOfPage":"https://obakeng.substack.com/p/its-a-log-eat-log-world","headline":"It's a log eat log world!","description":"Let's discuss logging - unstructured, structured and canonical log lines - what they are and what value they bring to your production systems.","image":[{"@type":"ImageObject","url":"https://substack-post-media.s3.amazonaws.com/public/images/a09059a6-f716-4e46-997e-037d21ff6ace_3636x2415.jpeg"}],"datePublished":"2025-02-07T13:30:40+00:00","dateModified":"2025-02-07T13:30:40+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"Obakeng Mosadi","url":"https://substack.com/@obakeng","description":"the weekly newsletter that makes reliability engineering accessible to anyone.","identifier":"user:75075413","image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!dVpg!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!dVpg!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg"}}],"publisher":{"@type":"Organization","name":"Unreliability.","url":"https://obakeng.substack.com","description":"Here we discuss reliability engineering and create experiments to prove ourselves right (or wrong).","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":100},"identifier":"pub:730597"},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":6},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":2},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":0}]}</script><script nonce="XYBzbX0dMnqLivGDUHjKMw" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Unreliability.","item":"https://obakeng.substack.com"},{"@type":"ListItem","position":2,"name":"It's a log eat log world!","item":"https://obakeng.substack.com/p/its-a-log-eat-log-world"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><h1 dir="auto" class="post-title published title-X77sOw">It's a log eat log world!</h1><h3 dir="auto" class="subtitle subtitle-HEEcLo">Let's discuss logging - unstructured, structured and canonical log lines - what they are and what value they bring to your production systems.</h3><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-row pc-gap-8 pc-alignItems-center pc-justifyContent-flex-start pc-reset"><div style="--scale:36px;--offset:9px;--border-width:4.5px;" class="pencraft pc-display-flex pc-flexDirection-row pc-alignItems-center pc-justifyContent-flex-start pc-reset ltr-qDBmby"><a href="https://substack.com/@obakeng" aria-label="View Obakeng Mosadi's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:36px;" tabindex="0" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6 avatar-u8q6xB last-JfNEJ_"><div style="--scale:36px;" title="Obakeng Mosadi" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!dVpg!,w_36,h_36,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg 1x, https://substackcdn.com/image/fetch/$s_!dVpg!,w_72,h_72,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg 2x, https://substackcdn.com/image/fetch/$s_!dVpg!,w_108,h_108,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg 3x"/><img src="https://substackcdn.com/image/fetch/$s_!dVpg!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg" alt="Obakeng Mosadi's avatar" srcset="https://substackcdn.com/image/fetch/$s_!dVpg!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg 1x, https://substackcdn.com/image/fetch/$s_!dVpg!,w_72,h_72,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg 2x, https://substackcdn.com/image/fetch/$s_!dVpg!,w_108,h_108,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg 3x" width="36" height="36" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-reset color-primary-zABazT line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><span style="min-width:0;" data-state="closed"><a href="https://substack.com/@obakeng" class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ">Obakeng Mosadi</a></span></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2025-02-07T13:30:40.276Z" title="2025-02-07T13:30:40.276Z">Feb 07, 2025</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (6)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">6</div></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://obakeng.substack.com/p/its-a-log-eat-log-world/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-27" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">2</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p>Following up on last week, I introduced the application we’ll work on: a mini-version of Substack.</p><p>This week, we start with logging. We discuss different kinds of logging, their benefits, and options for improving your log storage and retrieval.</p><p>Next, we design a logging architecture to transform our logs into actionable insights.</p><h3 class="header-anchor-post">Outline<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§outline" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/outline" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p>Introduction</p><ul><li><p>Importance of logs in modern systems</p></li></ul></li><li><p>What are structured logs?</p></li><li><p>What are canonical logs?</p></li><li><p>Challenges with log management</p><ul><li><p>Inefficient storage</p></li><li><p>Standardisation</p></li><li><p>Security</p></li><li><p>Irrelevant information</p></li></ul></li><li><p>Architecture for the demo app. </p><ul><li><p>Log structure</p></li></ul></li><li><p>Building a simple log data system with Clickhouse</p><ul><li><p>Schema design</p></li><li><p>Asking meaningful questions</p></li></ul></li><li><p>Conclusion</p></li></ul><h3 class="header-anchor-post">Importance of logging in modern systems<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§importance-of-logging-in-modern-systems" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/importance-of-logging-in-modern-systems" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Today's systems are more complex than ever. What used to be a single application running on a server is now multiple services communicating with each other over a network/s.</p><p>This can lead to system failures of all kinds, which can happen at any point. Knowing what happened during a request's lifecycle can help debug system failures and potential customer issues.</p><p>One way of understanding the lifecycle of a request is using logging. Emitting log events whenever something happens, storing them in a log store, and reaching for them in an on-call situation.</p><p>If your logs are well-structured, making correlations during incidents becomes considerably easier.</p><h4 class="header-anchor-post">Unstructured logs<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§unstructured-logs" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/unstructured-logs" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p>Let’s start with an example. You have an application that does some incredibly exciting stuff. You’ve built this really quickly because you want some customers onboard.</p><p> One day, your system goes down. You don’t know what happened, so you make assumptions and print something to standard output.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!lt4p!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!lt4p!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 424w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 848w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 1272w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!lt4p!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png" width="1456" height="126" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:126,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:50057,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!lt4p!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 424w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 848w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 1272w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p><br/><span>Then, you spend hours searching for the issue, which you eventually discover.</span></p><p>To simplify things for yourself, you implement unstructured logs—something that resembles this—and store it in a log file:</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!vuk2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!vuk2!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 424w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 848w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 1272w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!vuk2!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png" width="1456" height="99" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/f894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:99,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:81492,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!vuk2!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 424w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 848w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 1272w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p>An inconsistent log format characterizes unstructured logs. If you look at the example for unstructured logs above, you’ll see that there isn’t consistency with the way the logs are structured. And even though they have a timestamp placed on them, in some cases, you might find that some log events don’t even have a timestamp.</p><p>This format has a number of problems.</p><p>This can make it difficult to search and filter. You must use grep and awk to find what you’re looking for; it is not machine-readable.</p><p>Another issue is that, depending on how you store it, it can become expensive over time and may not be as beneficial for debugging.</p><p>On a tangent, here’s an example of logging that caused Kubernetes pods not to be scheduled:</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!Sztl!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!Sztl!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 424w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 848w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 1272w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!Sztl!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png" width="737" height="580" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:580,&quot;width&quot;:737,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:136354,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!Sztl!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 424w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 848w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 1272w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p>This post discusses solving problems with unstructured logs using structured and canonical logs.</p><h3 class="header-anchor-post">What are structured logs?<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§what-are-structured-logs" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/what-are-structured-logs" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>Structured logs are logs with a consistent format. Often outputting in formats like JSON or logfmt. They are designed to be machine-readable, which means they can be easily processed by log aggregators like Loki or ElasticSearch and follow a key-value pair structure.</span><br/><br/><span>This means parsing and analysing these logs becomes much easier using keys.</span><br/><br/><span>Here’s an example using </span><a href="https://www.structlog.org/en/24.4.0/getting-started.html">structlog</a><span>:</span></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!0usC!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!0usC!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 424w, https://substackcdn.com/image/fetch/$s_!0usC!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 848w, https://substackcdn.com/image/fetch/$s_!0usC!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 1272w, https://substackcdn.com/image/fetch/$s_!0usC!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!0usC!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png" width="1456" height="193" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/aba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:193,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:243598,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!0usC!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 424w, https://substackcdn.com/image/fetch/$s_!0usC!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 848w, https://substackcdn.com/image/fetch/$s_!0usC!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 1272w, https://substackcdn.com/image/fetch/$s_!0usC!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p>Here, you can see that there is a structure. A timestamped record with a log level, an event message and some metadata relating to the event in key-value pairs. In this example, I’ve decided to go with logfmt, but this can also be rendered in JSON, and it looks like this:</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!cYy3!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!cYy3!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 424w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 848w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 1272w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!cYy3!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png" width="1456" height="159" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:159,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:218341,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!cYy3!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 424w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 848w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 1272w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p><span>These formats are easier to search. I can filter these logs by a specific key, such as </span><em><strong>user_id</strong></em><span> or </span><em><strong>request_id</strong></em><span>. This allows you to “dump” any metadata in your logs, which will eventually be useful for diagnosing issues.</span></p><h3 class="header-anchor-post">What are canonical logs?<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§what-are-canonical-logs" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/what-are-canonical-logs" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Canonical logs are synonymous with structured logs. The key difference lies in that with canonical logs; you emit a single log line at the end of a request containing all the necessary data you might need. In contrast, standard structured logs emit multiple log lines per request. </p><p>The idea is to provide as much context as possible regarding a request with each log event.</p><p>Brandur Leach defines it like this:</p><blockquote><h4><strong>Literally, one big log line</strong></h4><p>The concept is simple: a canonical line is a big log line that gets emitted at the end of a request. It’s filled with fields for all of that request’s key information.</p></blockquote><p>Essentially, you could use canonical logs as a single source of truth about the events in your system because you dump all the context you might need for future investigation within your logs.</p><p>Structured and canonical logs also make for great analysis using tools like Clickhouse or DuckDB.</p><h3 class="header-anchor-post">Challenges with log management<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§challenges-with-log-management" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/challenges-with-log-management" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><h4 class="header-anchor-post">Inefficient storage<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§inefficient-storage" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/inefficient-storage" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p>One of the biggest challenges with log management is storage. The high volume of logs could become expensive because you emit multiple log lines per request over multiple handlers.</p><p>Of course, to handle this, you could implement retention policies to help improve the storage problem. For example, your system may only need seven days’ worth of logs for investigation, while other systems might need 90 days’ worth.</p><p>Another option is sampling, which involves capturing only a subset of logs instead of everything.</p><h4 class="header-anchor-post">Standardization<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§standardization" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/standardization" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p>Another challenge is standardizing the format of your logs. Every team might have its own way of formatting logs. Some might prefer logfmt, while others prefer JSON.</p><p>It’s important to follow a standardized format across your systems so that there’s a unified language.</p><h4 class="header-anchor-post">Security<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§security" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/security" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p>Another challenge is what kind of data to throw into the logs. It can happen that you dump sensitive data like personal user information like addresses, IDs, or passwords. Depending on your security posture, make sure you continue auditing for sensitive data in your logs.</p><h4 class="header-anchor-post">Irrelevant information<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§irrelevant-information" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/irrelevant-information" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p>One last challenge in managing logs is irrelevant information like verbose debug logs, logs with no context or repetitive messaging.</p><p>This can happen when logging events from the same handler multiple times.</p><p>Now, let’s move on to our demo app.</p><h3 class="header-anchor-post">The architecture<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§the-architecture" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/the-architecture" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Now that we understand logging, I’ll describe a simple architecture for our app.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!SPvn!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!SPvn!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 424w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 848w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 1272w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!SPvn!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png" width="1456" height="831" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:831,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:144806,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!SPvn!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 424w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 848w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 1272w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p></p><p><a href="https://vector.dev/">Vector</a><span> is a tool for building observability pipelines between our app and the log stores.</span></p><p>Vector collects our app logs and routes them to Loki, our log aggregator, and Clickhouse.</p><p>We want to be able to slice and dice our log data to ask meaningful questions about our system, so we send our events to Clickhouse as well and create visualisations with Grafana.</p><p>Another thing I do here is sample. Since I’m allowing Locust to spawn 5,000 users, I don’t want to end up with hundreds of thousands of logs, so I sample one out of 50 log events to keep things tidy.</p><p><span>Also, I fought long and hard to get Vector to work correctly with Loki and Clickhouse. If you want to see the pain, you can look at this </span><a href="https://github.com/obakeng-develops/writestack/pull/11">PR</a><span> to understand it. Figuring out the details was fun.</span></p><p>Much of my implementation is a combination of the ideas in these posts:</p><ul><li><p><a href="https://jeremymorrell.dev/blog/a-practitioners-guide-to-wide-events/">A Practitioner’s Guide to Wide Events</a></p></li><li><p><a href="https://boristane.com/blog/observability-wide-events-101/">Observability Wide Events 101</a></p></li><li><p><a href="https://brandur.org/canonical-log-lines">Canonical Log Lines</a></p></li><li><p><a href="https://charity.wtf/2024/08/07/is-it-time-to-version-observability-signs-point-to-yes/">Is It Time To Version Observability?</a></p></li><li><p><a href="https://thenewstack.io/the-4-evolutions-of-your-observability-journey/">The 4 Evolutions of Your Observability Journey</a></p></li><li><p><a href="https://isburmistrov.substack.com/p/all-you-need-is-wide-events-not-metrics">All you need is Wide Events</a></p></li></ul><p></p><h4 class="header-anchor-post">The log structure<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§the-log-structure" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/the-log-structure" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p><span>I’ve decided to use this structure for naming events: </span><em><strong>“resource.operation.status”.</strong></em><span> For example, </span><em><strong>“user.search.started”</strong></em><span> means the search for a user has begun. This helps me make a mental model of the type of operation that was happening in the system.</span></p><p><span>The opposite of this would be </span><em><strong>“user.search.failed”</strong></em><span>, meaning the search for a user failed.</span></p><p>I also add as much context about the event as possible, such as the request ID, hostname, user agent, database timings, user ID, framework version, language version, etc.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!APIY!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!APIY!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 424w, https://substackcdn.com/image/fetch/$s_!APIY!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 848w, https://substackcdn.com/image/fetch/$s_!APIY!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 1272w, https://substackcdn.com/image/fetch/$s_!APIY!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!APIY!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png" width="902" height="693" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/a7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:693,&quot;width&quot;:902,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:122177,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!APIY!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 424w, https://substackcdn.com/image/fetch/$s_!APIY!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 848w, https://substackcdn.com/image/fetch/$s_!APIY!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 1272w, https://substackcdn.com/image/fetch/$s_!APIY!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p>After running Locust, we get this result in Loki: the timestamp is on the far left, the unique labels are in the middle, and the structured logs are on the right.</p><p><span>Using the log structure we spoke about, looking at the above example, we can reason that our system successfully validated post data by looking at the </span><em><strong>“post.validation.success” </strong></em><span>event</span><em><strong> </strong></em><span>and then, right after the creation of the post, was successful.</span></p><p>This idea is to help an on-call engineer or anyone who is curious to understand what is happening. Of course, you don’t have to follow this structure but consider how you want to structure your logs to help anyone who is new to your log system.</p><h3 class="header-anchor-post">Designing a log system using Clickhouse<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§designing-a-log-system-using-clickhouse" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/designing-a-log-system-using-clickhouse" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>Part of the motivation comes from reading about Scuba and similar systems that enable users to navigate through data and ask questions about their systems. To achieve a similar capability, we can store our logs in Clickhouse.</span><br/><br/><span>So, to ensure that I design a sensible schema, I’ll use some of the information in the </span><a href="https://opentelemetry.io/docs/specs/otel/logs/data-model/">OTel Log Data Model</a><span> as a starting point.</span></p><p>My schema design looks like this:</p><pre><code>CREATE TABLE application_logs (
event String,
log_level String,
message String,
timestamp DateTime
) ENGINE = MergeTree()
ORDER BY timestamp;</code></pre><p>Originally, the schema mapped one-to-one to the fields in the logs. However, when I added new log fields, I had to create a new field in the table to accommodate them. That can be tedious.</p><p>This design, I think, is more flexible.</p><p>All the log messages are sent into the message field; if any changes are made, the table itself will not need to be changed.</p><p>Of course, this design is simple for our purposes but can be extended in whatever way your system requires. For example, if you needed to include information about traces.</p><p>Here’s the output of what the Clickhouse data looks like after generating a load on the app:</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!mYrS!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!mYrS!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 424w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 848w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 1272w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!mYrS!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png" width="1456" height="561" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/da53c1f5-594e-4935-8665-f9e621c46668_1696x654.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:561,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:149944,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!mYrS!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 424w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 848w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 1272w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><h4 class="header-anchor-post">Asking meaningful questions<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§asking-meaningful-questions" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/asking-meaningful-questions" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p>Ultimately, we want to be able to analyze our data and ask meaningful questions about our service. This will help us identify trends and subsequently build better services.</p><p>The one downside to this approach is that you need to have some idea of how to use SQL. Otherwise, it can be a drag.</p><p>So, let’s ask our first question.</p><h5>What is the count of successful user searches per hour?</h5><p>Here’s my SQL query for this question:</p><pre><code>SELECT
toStartOfHour(timestamp) AS hour,
count(*) AS search_count
FROM logs.application_logs
WHERE event = 'user.search.success'
GROUP BY hour
ORDER BY hour;</code></pre><p>Which results in:</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!4hjd!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!4hjd!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 424w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 848w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 1272w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!4hjd!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png" width="1456" height="397" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:397,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:137257,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!4hjd!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 424w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 848w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 1272w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p>Knowing at what periods of the day user searches happen can help us allocate resources during high-activity periods of the day/week/month.</p><p>Maybe we could ask the opposite.</p><h5>What is the count of unsuccessful user searches per hour?</h5><pre><code>SELECT
toStartOfHour(timestamp) AS hour,
count(*) AS search_count
FROM logs.application_logs
WHERE event = 'user.search.failed'
GROUP BY hour
ORDER BY hour;</code></pre><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!_4H5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!_4H5!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!_4H5!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png" width="1456" height="317" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:317,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:83997,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!_4H5!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p>Our system generally has fewer user search failures, but this could still indicate something that needs fixing.</p><p>Perhaps we can have a look at which device type has the most user search failures.</p><h5>What is the count of unsuccessful user searches per hour by device type?</h5><pre><code>SELECT
JSONExtractString(message, 'device_type') AS device_type,
count(*) AS search_count
FROM logs.application_logs
WHERE event = 'user.search.failed'
GROUP BY device_type
ORDER BY search_count DESC;</code></pre><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!0BEe!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!0BEe!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 424w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 848w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 1272w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!0BEe!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png" width="1456" height="357" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:357,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:87351,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!0BEe!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 424w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 848w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 1272w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p>This question can help us segment these user search failures by which devices they are coming from. In our example, load is generated by Locust, which uses Python’s request library, so we only have one device type showing on our dashboard.</p><p>In your use case, this could be a broader slice of data that you can use to diagnose which platforms/devices might be the most problematic.</p><p>We could ask which users had the most repeated failures on search.</p><h5>Which users are experiencing repeated failures?</h5><pre><code>SELECT
JSONExtractString(message, 'user_id') AS user_id,
count(*) AS failure_count
FROM logs.application_logs
WHERE event = 'user.search.failed'
GROUP BY user_id
HAVING failure_count > 3
ORDER BY failure_count DESC;</code></pre><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!6xvi!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!6xvi!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!6xvi!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png" width="1456" height="317" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:317,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:135579,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!6xvi!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p>We can use this to reach out to the top N users who are most affected proactively. For example, your use case might be an e-commerce store, and you might want to know which users had the most order or payment failures. </p><p>This kind of data from your logs can bring great insight into what your users are experiencing.</p><p>Now, say we wanted to know how many newsletters are being created per hour.</p><h5>What is the number of successfully created newsletters per hour?</h5><pre><code>SELECT
toStartOfHour(timestamp) AS hour,
count(*) AS action_count
FROM logs.application_logs
WHERE event = 'newsletter.creation.success'
GROUP BY hour
ORDER BY hour;</code></pre><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!YQnt!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!YQnt!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!YQnt!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png" width="1456" height="317" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:317,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:72123,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!YQnt!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p>This can help us track how often users create newsletters and at what times this occurs.</p><p>We could even investigate further to determine which users create the most newsletters. We could ask questions about newsletter subscriptions and which types of subscriptions (free or paid) are most popular on our app.</p><h3 class="header-anchor-post">Conclusion<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§conclusion" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://obakeng.substack.com/i/153407591/conclusion" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>This is by no means an exhaustive list of questions we can ask, but as you can see, using well-structured logs, we can easily debug issues in our systems when they arise.</p><p>We can become like the Business Intelligence teams and start interrogating our log data, taking proactive steps to resolve issues, and building better systems.</p><p>So, where do you start? You don’t have to design a system built on Clickhouse to gain insights into your data. You can start by structuring your logs.</p><p>Think about what kind of data you need and dump that context in your logs. If you’re worried about log volume being an issue, consider sampling and adjusting your retention periods.</p><p>If you get anything out of this post, it’s this: let’s build better, transparent systems.</p></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (6)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">6</div></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://obakeng.substack.com/p/its-a-log-eat-log-world/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-56" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">2</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-left"><path d="m12 19-7-7 7-7"></path><path d="M19 12H5"></path></svg>Previous</button><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o">Next<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></button></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="visibility-check"></div><div id="discussion" class="pencraft pc-display-flex pc-flexDirection-column pc-gap-16 pc-paddingTop-32 pc-paddingBottom-32 pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-32 pc-reset container"><h4 class="pencraft pc-reset line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Discussion about this post</h4><div class="pencraft pc-alignSelf-flex-start pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Select discussion type" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-70-content-comments" data-state="active" id="radix-P0-70-trigger-comments" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Comments</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-70-content-restacks" data-state="inactive" id="radix-P0-70-trigger-restacks" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Restacks</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div></div></div><div id="substack-comments" class="single-post-section comments-section"><div class="container"><div class="visibility-check"></div><div data-test-id="comment-input" class="pencraft pc-display-flex pc-reset flex-grow-rzmknG"><form class="comment-input-form form-CkZ7Kt"><div style="--scale:32px;" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><div style="--scale:32px;" title="User" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-flexDirection-column pc-boxShadow-xs pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-sm"><textarea name="body" placeholder="Write a comment..." aria-label="Write a comment..." rows="4" class="pencraft input-qHk4bN autogrowing-_ipn9Y textarea-GbEjRX inputText-pV_yWb unstyled-IJLUJU"></textarea></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></form></div></div></div></div><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-83-content-top" data-state="active" id="radix-P0-83-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-83-content-new" data-state="inactive" id="radix-P0-83-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer themed-background"><div class="container"><div class="footer-terms"><span>© 2026 Obakeng</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&amp;utm_medium=web&amp;utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&amp;utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="XYBzbX0dMnqLivGDUHjKMw">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
</div>
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
<script nonce="XYBzbX0dMnqLivGDUHjKMw">
window.Sentry && window.Sentry.onLoad(function() {
window.Sentry.init({
environment: window._preloads.sentry_environment,
dsn: window._preloads.sentry_dsn,
})
})
</script>
<script nonce="XYBzbX0dMnqLivGDUHjKMw">window._preloads = JSON.parse("{\"cspNonce\":\"XYBzbX0dMnqLivGDUHjKMw\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://obakeng.substack.com\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":75075413,\"byline_images_enabled\":true,\"bylines_enabled\":true,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Obakeng\",\"cover_photo_url\":null,\"created_at\":\"2022-02-04T14:22:04.122Z\",\"custom_domain_optional\":false,\"custom_domain\":null,\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":null,\"default_show_guest_bios\":true,\"email_banner_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/efd99d5e-9f2f-483f-85c4-e1925b89f712_1100x220.png\",\"email_from_name\":null,\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":null,\"free_subscription_benefits\":null,\"ga_pixel_id\":null,\"google_site_verification_token\":null,\"google_tag_manager_token\":null,\"hero_image\":null,\"hero_text\":\"Here we discuss reliability engineering and create experiments to prove ourselves right (or wrong).\",\"hide_intro_subtitle\":null,\"hide_intro_title\":null,\"hide_podcast_feed_link\":false,\"homepage_type\":\"newspaper\",\"id\":730597,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":null,\"logo_url\":null,\"minimum_group_size\":2,\"moderation_enabled\":true,\"name\":\"Unreliability.\",\"paid_subscription_benefits\":null,\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"disabled\",\"paywall_free_trial_enabled\":false,\"podcast_art_url\":null,\"paid_podcast_episode_art_url\":null,\"podcast_byline\":null,\"podcast_description\":null,\"podcast_enabled\":false,\"podcast_feed_url\":null,\"podcast_title\":null,\"post_preview_limit\":null,\"primary_user_id\":75075413,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"obakeng\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#786CFF\",\"theme_var_color_links\":false,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":null,\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":null,\"stripe_user_id\":null,\"stripe_country\":null,\"stripe_publishable_key\":null,\"stripe_platform_account\":null,\"automatic_tax_enabled\":null,\"author_name\":\"Obakeng Mosadi\",\"author_handle\":\"obakeng\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!dVpg!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg\",\"author_bio\":\"the weekly newsletter that makes reliability engineering accessible to anyone.\",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":\"#e11d48\",\"web_bg_color\":\"#ffffff\",\"cover_bg_color\":null,\"publication_id\":730597,\"color_links\":null,\"font_preset_heading\":\"sans\",\"font_preset_body\":\"mono\",\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"feature-media\",\"home_posts\":\"list\",\"home_show_top_posts\":true,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":null,\"custom_footer\":null,\"social_media_links\":null,\"font_options\":null,\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":{\"first_thread_email_sent_at\":null,\"activated_at\":null,\"reader_thread_notifications_enabled\":false,\"create_thread_minimum_role\":\"contributor\",\"photo_replies_enabled\":true,\"boost_free_subscriber_chat_preview_enabled\":false,\"push_suppression_enabled\":false},\"default_group_coupon_percent_off\":null,\"default_group_coupon_include_founding\":null,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2022-04-28T07:34:36.449Z\",\"has_podcast\":false,\"has_free_podcast\":false,\"has_subscriber_only_podcast\":false,\"has_community_content\":false,\"rankingDetail\":\"Launched 4 years ago\",\"rankingDetailFreeIncluded\":\"Hundreds of subscribers\",\"rankingDetailOrderOfMagnitude\":0,\"rankingDetailFreeIncludedOrderOfMagnitude\":100,\"rankingDetailFreeSubscriberCount\":null,\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u062A\u0645 \u0627\u0644\u0625\u0637\u0644\u0627\u0642 4 years ago\"},\"ca\":{\"rankingDetail\":\"S\u2019ha llan\u00E7at fa 4 anys\"},\"da\":{\"rankingDetail\":\"Lancering 4 \u00E5r\"},\"de\":{\"rankingDetail\":\"vor 4 Jahren gelauncht\"},\"es\":{\"rankingDetail\":\"Lanzado hace 4 a\u00F1os\"},\"fr\":{\"rankingDetail\":\"Lanc\u00E9 il y a 4 ann\u00E9es\"},\"id\":{\"rankingDetail\":\"Diluncurkan 4 tahun yang lalu\"},\"ja\":{\"rankingDetail\":\"\u958B\u59CB\u65E5 4\u5E74\u524D\"},\"ko\":{\"rankingDetail\":\"4\uB144 \uC804\uC5D0 \uCD9C\uC2DC\"},\"nb\":{\"rankingDetail\":\"Lansert 4 \u00E5r\"},\"nl\":{\"rankingDetail\":\"Gelanceerd 4 jaar geleden\"},\"pl\":{\"rankingDetail\":\"Uruchomiono 4 lat temu\"},\"pt\":{\"rankingDetail\":\"Lan\u00E7ado 4 anos\"},\"pt-br\":{\"rankingDetail\":\"Lan\u00E7ado 4 anos\"},\"en-gb\":{\"rankingDetail\":\"Launched 4 years ago\"},\"it\":{\"rankingDetail\":\"Lanciato 4 anni\"},\"tr\":{\"rankingDetail\":\"4 y\u0131l ba\u015Flat\u0131ld\u0131\"},\"sv\":{\"rankingDetail\":\"Lanserad 4 \u00E5r sedan\"},\"fi\":{\"rankingDetail\":\"Launched 4 vuotta\"},\"is\":{\"rankingDetail\":\"Launched 4 \u00E1r\"},\"zh-hans\":{\"rankingDetail\":\"\u4E8E4 \u5E74\u524D\u4E0A\u7EBF\"},\"zh-hant\":{\"rankingDetail\":\"\u65BC 4 \u5E74\u524D \u63A8\u51FA\"},\"en\":{\"rankingDetail\":\"Launched 4 years ago\"}},\"freeSubscriberCount\":null,\"freeSubscriberCountOrderOfMagnitude\":\"186\",\"author_bestseller_tier\":0,\"author_badge\":null,\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":false,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":null,\"primary_profile_name\":\"Obakeng Mosadi\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!dVpg!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg\",\"no_follow\":false,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"free\",\"sections\":[],\"podcastTabInfo\":null,\"multipub_migration\":null,\"navigationBarItems\":[],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Obakeng Mosadi\",\"handle\":\"obakeng\",\"role\":\"admin\",\"owner\":true,\"user_id\":75075413,\"photo_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg\",\"bio\":\"the weekly newsletter that makes reliability engineering accessible to anyone.\",\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"threads_v2_enabled\":true,\"viralGiftsConfig\":null,\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://obakeng.substack.com\",\"hostname\":\"obakeng.substack.com\",\"is_on_substack\":false,\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"pageThemes\":{\"podcast\":null},\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#e11d48\",\"primary\":{\"r\":225,\"g\":29,\"b\":72,\"a\":1},\"primary_hover\":{\"r\":203,\"g\":0,\"b\":56,\"a\":1},\"primary_elevated\":{\"r\":203,\"g\":0,\"b\":56,\"a\":1},\"secondary\":{\"r\":225,\"g\":29,\"b\":72,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":225,\"g\":29,\"b\":72,\"a\":0.2},\"bg_hover\":{\"r\":225,\"g\":29,\"b\":72,\"a\":0.3},\"dark\":{\"primary\":{\"r\":225,\"g\":29,\"b\":72,\"a\":1},\"primary_hover\":{\"r\":247,\"g\":59,\"b\":88,\"a\":1},\"primary_elevated\":{\"r\":247,\"g\":59,\"b\":88,\"a\":1},\"secondary\":{\"r\":225,\"g\":29,\"b\":72,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":225,\"g\":29,\"b\":72,\"a\":0.2},\"bg_hover\":{\"r\":225,\"g\":29,\"b\":72,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":225,\"g\":29,\"b\":72,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":241,\"g\":51,\"b\":84,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":206.90096477355226,\"g\":206.90096477355175,\"b\":206.9009647735519,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#e11d48\",\"primary\":{\"r\":225,\"g\":29,\"b\":72,\"a\":1},\"primary_hover\":{\"r\":202,\"g\":26,\"b\":65,\"a\":1},\"primary_elevated\":{\"r\":225,\"g\":29,\"b\":72,\"a\":1},\"secondary\":{\"r\":225,\"g\":29,\"b\":72,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":225,\"g\":29,\"b\":72,\"a\":1},\"primary_hover\":{\"r\":247,\"g\":59,\"b\":88,\"a\":1},\"primary_elevated\":{\"r\":247,\"g\":59,\"b\":88,\"a\":1},\"secondary\":{\"r\":225,\"g\":29,\"b\":72,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":225,\"g\":29,\"b\":72,\"a\":0.2},\"bg_hover\":{\"r\":225,\"g\":29,\"b\":72,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"secondary\":{\"r\":134,\"g\":135,\"b\":135,\"a\":1},\"tertiary\":{\"r\":146,\"g\":146,\"b\":146,\"a\":1},\"accent\":{\"r\":225,\"g\":29,\"b\":72,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":241,\"g\":51,\"b\":84,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"secondary\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"secondary_elevated\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"tertiary\":{\"r\":221,\"g\":221,\"b\":221,\"a\":1},\"quaternary\":{\"r\":183,\"g\":183,\"b\":183,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}},\"wordmark_bg\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"is_dark\":false},\"fonts\":{\"heading\":\"sans\",\"body\":\"mono\"}}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":true,\"customDomain\":null},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":false,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":false,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":null,\"twitterPixelId\":null,\"parselyPixelId\":null,\"googleTagManagerToken\":null,\"googleAnalytics4Token\":null,\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":true,\"credit_token_enabled\":false,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":false,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":true,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":null,\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"Unreliability is a newsletter about reliability engineering. We discuss observability, system design and chaos engineering. \",\"notes_feed_enabled\":false,\"paywall_unlock_tokens\":false,\"post_preview_crop_gravity\":\"center\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":false,\"reader_referrals_leaderboard_enabled\":false,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":false,\"local_currency_modal_seen\":false,\"local_payment_methods_modal_seen\":false,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":false,\"welcome_page_opt_out_text\":\"No thanks\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":false,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://obakeng.substack.com/p/its-a-log-eat-log-world\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":153407591,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2025-02-07T13:30:40.276Z\",\"updated_at\":\"2025-02-08T05:55:11.270Z\",\"publication_id\":730597,\"search_engine_description\":null,\"search_engine_title\":null,\"section_id\":null,\"should_send_free_preview\":false,\"show_guest_bios\":true,\"slug\":\"its-a-log-eat-log-world\",\"social_title\":null,\"subtitle\":\"Let's discuss logging - unstructured, structured and canonical log lines - what they are and what value they bring to your production systems.\",\"teaser_post_eligible\":true,\"title\":\"It's a log eat log world!\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"everyone\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":2,\"reactions\":{\"\u2764\":6},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"getting-started-with-observability\",\"next_post_slug\":\"zdk-building-a-5-minute-local-development\",\"cover_image\":\"https://substack-post-media.s3.amazonaws.com/public/images/a09059a6-f716-4e46-997e-037d21ff6ace_3636x2415.jpeg\",\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":{\"post_id\":153407591,\"podcast_episode_number\":null,\"podcast_season_number\":null,\"podcast_episode_type\":null,\"should_syndicate_to_other_feed\":null,\"syndicate_to_section_id\":null,\"hide_from_feed\":false,\"free_podcast_url\":null,\"free_podcast_duration\":null,\"preview_contains_ad\":false,\"was_imported_self_serve_sync\":false,\"draft_free_podcast_url\":null,\"draft_free_podcast_duration\":null},\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"Let's discuss logging - unstructured, structured and canonical log lines - what they are and what value they bring to your production systems.\",\"body_html\":\"<p>Following up on last week, I introduced the application we\u2019ll work on: a mini-version of Substack.</p><p>This week, we start with logging. We discuss different kinds of logging, their benefits, and options for improving your log storage and retrieval.</p><p>Next, we design a logging architecture to transform our logs into actionable insights.</p><h3>Outline</h3><ul><li><p>Introduction</p><ul><li><p>Importance of logs in modern systems</p></li></ul></li><li><p>What are structured logs?</p></li><li><p>What are canonical logs?</p></li><li><p>Challenges with log management</p><ul><li><p>Inefficient storage</p></li><li><p>Standardisation</p></li><li><p>Security</p></li><li><p>Irrelevant information</p></li></ul></li><li><p>Architecture for the demo app. </p><ul><li><p>Log structure</p></li></ul></li><li><p>Building a simple log data system with Clickhouse</p><ul><li><p>Schema design</p></li><li><p>Asking meaningful questions</p></li></ul></li><li><p>Conclusion</p></li></ul><h3>Importance of logging in modern systems</h3><p>Today's systems are more complex than ever. What used to be a single application running on a server is now multiple services communicating with each other over a network/s.</p><p>This can lead to system failures of all kinds, which can happen at any point. Knowing what happened during a request's lifecycle can help debug system failures and potential customer issues.</p><p>One way of understanding the lifecycle of a request is using logging. Emitting log events whenever something happens, storing them in a log store, and reaching for them in an on-call situation.</p><p>If your logs are well-structured, making correlations during incidents becomes considerably easier.</p><h4>Unstructured logs</h4><p>Let\u2019s start with an example. You have an application that does some incredibly exciting stuff. You\u2019ve built this really quickly because you want some customers onboard.</p><p> One day, your system goes down. You don\u2019t know what happened, so you make assumptions and print something to standard output.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!lt4p!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!lt4p!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 424w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 848w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 1272w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!lt4p!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png\\\" width=\\\"1456\\\" height=\\\"126\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:126,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:50057,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!lt4p!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 424w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 848w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 1272w, https://substackcdn.com/image/fetch/$s_!lt4p!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1bc2aea8-4afc-479f-8032-c9f053a12ec8_2499x217.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p><br>Then, you spend hours searching for the issue, which you eventually discover.</p><p>To simplify things for yourself, you implement unstructured logs\u2014something that resembles this\u2014and store it in a log file:</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!vuk2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!vuk2!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 424w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 848w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 1272w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!vuk2!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png\\\" width=\\\"1456\\\" height=\\\"99\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/f894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:99,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:81492,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!vuk2!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 424w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 848w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 1272w, https://substackcdn.com/image/fetch/$s_!vuk2!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff894577a-f794-4b0e-9faa-81edfc796fd7_3050x207.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>An inconsistent log format characterizes unstructured logs. If you look at the example for unstructured logs above, you\u2019ll see that there isn\u2019t consistency with the way the logs are structured. And even though they have a timestamp placed on them, in some cases, you might find that some log events don\u2019t even have a timestamp.</p><p>This format has a number of problems.</p><p>This can make it difficult to search and filter. You must use grep and awk to find what you\u2019re looking for; it is not machine-readable.</p><p>Another issue is that, depending on how you store it, it can become expensive over time and may not be as beneficial for debugging.</p><p>On a tangent, here\u2019s an example of logging that caused Kubernetes pods not to be scheduled:</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!Sztl!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!Sztl!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 424w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 848w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 1272w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!Sztl!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png\\\" width=\\\"737\\\" height=\\\"580\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:580,&quot;width&quot;:737,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:136354,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!Sztl!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 424w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 848w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 1272w, https://substackcdn.com/image/fetch/$s_!Sztl!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9a9c06a7-1fad-4124-9967-56fd2f65fc0d_737x580.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p>This post discusses solving problems with unstructured logs using structured and canonical logs.</p><h3>What are structured logs?</h3><p>Structured logs are logs with a consistent format. Often outputting in formats like JSON or logfmt. They are designed to be machine-readable, which means they can be easily processed by log aggregators like Loki or ElasticSearch and follow a key-value pair structure.<br><br>This means parsing and analysing these logs becomes much easier using keys.<br><br>Here\u2019s an example using <a href=\\\"https://www.structlog.org/en/24.4.0/getting-started.html\\\">structlog</a>:</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!0usC!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!0usC!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 424w, https://substackcdn.com/image/fetch/$s_!0usC!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 848w, https://substackcdn.com/image/fetch/$s_!0usC!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 1272w, https://substackcdn.com/image/fetch/$s_!0usC!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!0usC!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png\\\" width=\\\"1456\\\" height=\\\"193\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/aba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:193,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:243598,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!0usC!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 424w, https://substackcdn.com/image/fetch/$s_!0usC!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 848w, https://substackcdn.com/image/fetch/$s_!0usC!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 1272w, https://substackcdn.com/image/fetch/$s_!0usC!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faba2c54f-e733-4b7b-a10d-89c9973f3dbc_2639x349.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>Here, you can see that there is a structure. A timestamped record with a log level, an event message and some metadata relating to the event in key-value pairs. In this example, I\u2019ve decided to go with logfmt, but this can also be rendered in JSON, and it looks like this:</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!cYy3!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!cYy3!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 424w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 848w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 1272w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!cYy3!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png\\\" width=\\\"1456\\\" height=\\\"159\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:159,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:218341,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!cYy3!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 424w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 848w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 1272w, https://substackcdn.com/image/fetch/$s_!cYy3!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F476f10cf-9001-4dd3-8090-65ff21fb0bd6_3255x355.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>These formats are easier to search. I can filter these logs by a specific key, such as <em><strong>user_id</strong></em> or <em><strong>request_id</strong></em>. This allows you to \u201Cdump\u201D any metadata in your logs, which will eventually be useful for diagnosing issues.</p><h3>What are canonical logs?</h3><p>Canonical logs are synonymous with structured logs. The key difference lies in that with canonical logs; you emit a single log line at the end of a request containing all the necessary data you might need. In contrast, standard structured logs emit multiple log lines per request. </p><p>The idea is to provide as much context as possible regarding a request with each log event.</p><p>Brandur Leach defines it like this:</p><blockquote><h4><strong>Literally, one big log line</strong></h4><p>The concept is simple: a canonical line is a big log line that gets emitted at the end of a request. It\u2019s filled with fields for all of that request\u2019s key information.</p></blockquote><p>Essentially, you could use canonical logs as a single source of truth about the events in your system because you dump all the context you might need for future investigation within your logs.</p><p>Structured and canonical logs also make for great analysis using tools like Clickhouse or DuckDB.</p><h3>Challenges with log management</h3><h4>Inefficient storage</h4><p>One of the biggest challenges with log management is storage. The high volume of logs could become expensive because you emit multiple log lines per request over multiple handlers.</p><p>Of course, to handle this, you could implement retention policies to help improve the storage problem. For example, your system may only need seven days\u2019 worth of logs for investigation, while other systems might need 90 days\u2019 worth.</p><p>Another option is sampling, which involves capturing only a subset of logs instead of everything.</p><h4>Standardization</h4><p>Another challenge is standardizing the format of your logs. Every team might have its own way of formatting logs. Some might prefer logfmt, while others prefer JSON.</p><p>It\u2019s important to follow a standardized format across your systems so that there\u2019s a unified language.</p><h4>Security</h4><p>Another challenge is what kind of data to throw into the logs. It can happen that you dump sensitive data like personal user information like addresses, IDs, or passwords. Depending on your security posture, make sure you continue auditing for sensitive data in your logs.</p><h4>Irrelevant information</h4><p>One last challenge in managing logs is irrelevant information like verbose debug logs, logs with no context or repetitive messaging.</p><p>This can happen when logging events from the same handler multiple times.</p><p>Now, let\u2019s move on to our demo app.</p><h3>The architecture</h3><p>Now that we understand logging, I\u2019ll describe a simple architecture for our app.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!SPvn!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!SPvn!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 424w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 848w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 1272w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!SPvn!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png\\\" width=\\\"1456\\\" height=\\\"831\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:831,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:144806,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!SPvn!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 424w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 848w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 1272w, https://substackcdn.com/image/fetch/$s_!SPvn!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F30a2acd7-e9d2-4e08-a35a-8fa187330eb9_2816x1608.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p></p><p><a href=\\\"https://vector.dev/\\\">Vector</a>&nbsp;is a tool for building observability pipelines between our app and the log stores.</p><p>Vector collects our app logs and routes them to Loki, our log aggregator, and Clickhouse.</p><p>We want to be able to slice and dice our log data to ask meaningful questions about our system, so we send our events to Clickhouse as well and create visualisations with Grafana.</p><p>Another thing I do here is sample. Since I\u2019m allowing Locust to spawn 5,000 users, I don\u2019t want to end up with hundreds of thousands of logs, so I sample one out of 50 log events to keep things tidy.</p><p>Also, I fought long and hard to get Vector to work correctly with Loki and Clickhouse. If you want to see the pain, you can look at this&nbsp;<a href=\\\"https://github.com/obakeng-develops/writestack/pull/11\\\">PR</a>&nbsp;to understand it. Figuring out the details was fun.</p><p>Much of my implementation is a combination of the ideas in these posts:</p><ul><li><p><a href=\\\"https://jeremymorrell.dev/blog/a-practitioners-guide-to-wide-events/\\\">A Practitioner\u2019s Guide to Wide Events</a></p></li><li><p><a href=\\\"https://boristane.com/blog/observability-wide-events-101/\\\">Observability Wide Events 101</a></p></li><li><p><a href=\\\"https://brandur.org/canonical-log-lines\\\">Canonical Log Lines</a></p></li><li><p><a href=\\\"https://charity.wtf/2024/08/07/is-it-time-to-version-observability-signs-point-to-yes/\\\">Is It Time To Version Observability?</a></p></li><li><p><a href=\\\"https://thenewstack.io/the-4-evolutions-of-your-observability-journey/\\\">The 4 Evolutions of Your Observability Journey</a></p></li><li><p><a href=\\\"https://isburmistrov.substack.com/p/all-you-need-is-wide-events-not-metrics\\\">All you need is Wide Events</a></p></li></ul><p></p><h4>The log structure</h4><p>I\u2019ve decided to use this structure for naming events: <em><strong>\u201Cresource.operation.status\u201D.</strong></em> For example, <em><strong>\u201Cuser.search.started\u201D</strong></em> means the search for a user has begun. This helps me make a mental model of the type of operation that was happening in the system.</p><p>The opposite of this would be&nbsp;<em><strong>\u201Cuser.search.failed\u201D</strong></em>, meaning the search for a user failed.</p><p>I also add as much context about the event as possible, such as the request ID, hostname, user agent, database timings, user ID, framework version, language version, etc.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!APIY!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!APIY!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 424w, https://substackcdn.com/image/fetch/$s_!APIY!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 848w, https://substackcdn.com/image/fetch/$s_!APIY!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 1272w, https://substackcdn.com/image/fetch/$s_!APIY!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!APIY!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png\\\" width=\\\"902\\\" height=\\\"693\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/a7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:693,&quot;width&quot;:902,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:122177,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!APIY!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 424w, https://substackcdn.com/image/fetch/$s_!APIY!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 848w, https://substackcdn.com/image/fetch/$s_!APIY!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 1272w, https://substackcdn.com/image/fetch/$s_!APIY!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa7e9652e-26c2-4ed2-a66b-65b57d92f477_902x693.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p>After running Locust, we get this result in Loki: the timestamp is on the far left, the unique labels are in the middle, and the structured logs are on the right.</p><p>Using the log structure we spoke about, looking at the above example, we can reason that our system successfully validated post data by looking at the&nbsp;<em><strong>\u201Cpost.validation.success\u201D&nbsp;</strong></em>event<em><strong>&nbsp;</strong></em>and then, right after the creation of the post, was successful.</p><p>This idea is to help an on-call engineer or anyone who is curious to understand what is happening. Of course, you don\u2019t have to follow this structure but consider how you want to structure your logs to help anyone who is new to your log system.</p><h3>Designing a log system using Clickhouse</h3><p>Part of the motivation comes from reading about Scuba and similar systems that enable users to navigate through data and ask questions about their systems. To achieve a similar capability, we can store our logs in Clickhouse.<br><br>So, to ensure that I design a sensible schema, I\u2019ll use some of the information in the&nbsp;<a href=\\\"https://opentelemetry.io/docs/specs/otel/logs/data-model/\\\">OTel Log Data Model</a>&nbsp;as a starting point.</p><p>My schema design looks like this:</p><pre><code>CREATE TABLE application_logs (\\n\\tevent String,\\n\\tlog_level String,\\n\\tmessage String,\\n\\ttimestamp DateTime\\n) ENGINE = MergeTree()\\nORDER BY timestamp;</code></pre><p>Originally, the schema mapped one-to-one to the fields in the logs. However, when I added new log fields, I had to create a new field in the table to accommodate them. That can be tedious.</p><p>This design, I think, is more flexible.</p><p>All the log messages are sent into the message field; if any changes are made, the table itself will not need to be changed.</p><p>Of course, this design is simple for our purposes but can be extended in whatever way your system requires. For example, if you needed to include information about traces.</p><p>Here\u2019s the output of what the Clickhouse data looks like after generating a load on the app:</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!mYrS!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!mYrS!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 424w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 848w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 1272w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!mYrS!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png\\\" width=\\\"1456\\\" height=\\\"561\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/da53c1f5-594e-4935-8665-f9e621c46668_1696x654.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:561,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:149944,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!mYrS!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 424w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 848w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 1272w, https://substackcdn.com/image/fetch/$s_!mYrS!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fda53c1f5-594e-4935-8665-f9e621c46668_1696x654.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><h4>Asking meaningful questions</h4><p>Ultimately, we want to be able to analyze our data and ask meaningful questions about our service. This will help us identify trends and subsequently build better services.</p><p>The one downside to this approach is that you need to have some idea of how to use SQL. Otherwise, it can be a drag.</p><p>So, let\u2019s ask our first question.</p><h5>What is the count of successful user searches per hour?</h5><p>Here\u2019s my SQL query for this question:</p><pre><code>SELECT\\n toStartOfHour(timestamp) AS hour,\\n count(*) AS search_count\\nFROM logs.application_logs\\nWHERE event = 'user.search.success'\\nGROUP BY hour\\nORDER BY hour;</code></pre><p>Which results in:</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!4hjd!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!4hjd!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 424w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 848w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 1272w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!4hjd!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png\\\" width=\\\"1456\\\" height=\\\"397\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:397,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:137257,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!4hjd!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 424w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 848w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 1272w, https://substackcdn.com/image/fetch/$s_!4hjd!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F4d490976-bc95-46d3-ae3e-1b6189c618ca_2760x752.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p>Knowing at what periods of the day user searches happen can help us allocate resources during high-activity periods of the day/week/month.</p><p>Maybe we could ask the opposite.</p><h5>What is the count of unsuccessful user searches per hour?</h5><pre><code>SELECT\\n toStartOfHour(timestamp) AS hour,\\n count(*) AS search_count\\nFROM logs.application_logs\\nWHERE event = 'user.search.failed'\\nGROUP BY hour\\nORDER BY hour;</code></pre><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!_4H5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!_4H5!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!_4H5!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png\\\" width=\\\"1456\\\" height=\\\"317\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:317,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:83997,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!_4H5!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!_4H5!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F12233e16-7e17-445a-876d-ce526a973fc8_2760x600.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>Our system generally has fewer user search failures, but this could still indicate something that needs fixing.</p><p>Perhaps we can have a look at which device type has the most user search failures.</p><h5>What is the count of unsuccessful user searches per hour by device type?</h5><pre><code>SELECT\\n JSONExtractString(message, 'device_type') AS device_type,\\n count(*) AS search_count\\nFROM logs.application_logs\\nWHERE event = 'user.search.failed'\\nGROUP BY device_type\\nORDER BY search_count DESC;</code></pre><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!0BEe!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!0BEe!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 424w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 848w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 1272w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!0BEe!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png\\\" width=\\\"1456\\\" height=\\\"357\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:357,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:87351,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!0BEe!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 424w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 848w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 1272w, https://substackcdn.com/image/fetch/$s_!0BEe!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F704191c0-9afe-4147-9c53-41a526fcdc0e_2760x676.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>This question can help us segment these user search failures by which devices they are coming from. In our example, load is generated by Locust, which uses Python\u2019s request library, so we only have one device type showing on our dashboard.</p><p>In your use case, this could be a broader slice of data that you can use to diagnose which platforms/devices might be the most problematic.</p><p>We could ask which users had the most repeated failures on search.</p><h5>Which users are experiencing repeated failures?</h5><pre><code>SELECT\\n JSONExtractString(message, 'user_id') AS user_id,\\n count(*) AS failure_count\\nFROM logs.application_logs\\nWHERE event = 'user.search.failed'\\nGROUP BY user_id\\nHAVING failure_count &gt; 3\\nORDER BY failure_count DESC;</code></pre><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!6xvi!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!6xvi!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!6xvi!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png\\\" width=\\\"1456\\\" height=\\\"317\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:317,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:135579,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!6xvi!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!6xvi!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F99863d59-6280-453a-a63d-76df1ee50a07_2760x600.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>We can use this to reach out to the top N users who are most affected proactively. For example, your use case might be an e-commerce store, and you might want to know which users had the most order or payment failures. </p><p>This kind of data from your logs can bring great insight into what your users are experiencing.</p><p>Now, say we wanted to know how many newsletters are being created per hour.</p><h5>What is the number of successfully created newsletters per hour?</h5><pre><code>SELECT\\n toStartOfHour(timestamp) AS hour,\\n count(*) AS action_count\\nFROM logs.application_logs\\nWHERE event = 'newsletter.creation.success'\\nGROUP BY hour\\nORDER BY hour;</code></pre><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!YQnt!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!YQnt!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!YQnt!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png\\\" width=\\\"1456\\\" height=\\\"317\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:317,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:72123,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!YQnt!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 424w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 848w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 1272w, https://substackcdn.com/image/fetch/$s_!YQnt!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F09f417c6-11d8-4f54-85bb-85fa3b84c718_2760x600.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p>This can help us track how often users create newsletters and at what times this occurs.</p><p>We could even investigate further to determine which users create the most newsletters. We could ask questions about newsletter subscriptions and which types of subscriptions (free or paid) are most popular on our app.</p><h3>Conclusion</h3><p>This is by no means an exhaustive list of questions we can ask, but as you can see, using well-structured logs, we can easily debug issues in our systems when they arise.</p><p>We can become like the Business Intelligence teams and start interrogating our log data, taking proactive steps to resolve issues, and building better systems.</p><p>So, where do you start? You don\u2019t have to design a system built on Clickhouse to gain insights into your data. You can start by structuring your logs.</p><p>Think about what kind of data you need and dump that context in your logs. If you\u2019re worried about log volume being an issue, consider sampling and adjusting your retention periods.</p><p>If you get anything out of this post, it\u2019s this: let\u2019s build better, transparent systems.</p>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"Following up on last week, I introduced the application we\u2019ll work on: a mini-version of Substack.\",\"wordcount\":2196,\"post_preview_limit\":null,\"language\":\"en\",\"postTags\":[{\"id\":\"0e922f71-62ee-40f6-bcb4-9a63177a1ba7\",\"publication_id\":730597,\"name\":\"reliability\",\"slug\":\"reliability\",\"hidden\":false},{\"id\":\"129ebfcb-4ea6-43ac-9ad9-87475b27aac9\",\"publication_id\":730597,\"name\":\"aws\",\"slug\":\"aws\",\"hidden\":false},{\"id\":\"351b1698-e828-4123-a2c6-8d2a5c108bee\",\"publication_id\":730597,\"name\":\"observability\",\"slug\":\"observability\",\"hidden\":false},{\"id\":\"71404427-89f2-4440-9696-78459d9fef55\",\"publication_id\":730597,\"name\":\"devops\",\"slug\":\"devops\",\"hidden\":false},{\"id\":\"a5126cf9-4cf8-4553-b4e3-617f7295d4b0\",\"publication_id\":730597,\"name\":\"gcp\",\"slug\":\"gcp\",\"hidden\":false},{\"id\":\"b0645f48-cd42-4cc2-aec3-37d455e1e604\",\"publication_id\":730597,\"name\":\"cloud\",\"slug\":\"cloud\",\"hidden\":false},{\"id\":\"cb5c13ec-e704-4464-a5c5-5e0532d9f516\",\"publication_id\":730597,\"name\":\"azure\",\"slug\":\"azure\",\"hidden\":false},{\"id\":\"e804e312-a815-4ce7-86bb-e0324d2fcfaa\",\"publication_id\":730597,\"name\":\"clickhouse\",\"slug\":\"clickhouse\",\"hidden\":false},{\"id\":\"faeccb3c-7276-4f90-a77d-aff486744955\",\"publication_id\":730597,\"name\":\"logs\",\"slug\":\"logs\",\"hidden\":false}],\"postCountryBlocks\":[],\"headlineTest\":null,\"coverImagePalette\":{\"Vibrant\":{\"rgb\":[163,109,73],\"population\":628},\"DarkVibrant\":{\"rgb\":[101,62,40],\"population\":615},\"LightVibrant\":{\"rgb\":[220,192,180],\"population\":100},\"Muted\":{\"rgb\":[159,128,119],\"population\":256},\"DarkMuted\":{\"rgb\":[115,76,50],\"population\":181},\"LightMuted\":{\"rgb\":[206,178,164],\"population\":117}},\"publishedBylines\":[{\"id\":75075413,\"name\":\"Obakeng Mosadi\",\"handle\":\"obakeng\",\"photo_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/82c51225-354c-4cae-8f93-a07311c0ba6b_800x800.jpeg\",\"bio\":\"the weekly newsletter that makes reliability engineering accessible to anyone.\",\"profile_set_up_at\":\"2022-09-23T10:25:19.756Z\",\"reader_installed_at\":\"2024-11-02T08:14:32.189Z\",\"publicationUsers\":[{\"id\":665980,\"user_id\":75075413,\"publication_id\":730597,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":730597,\"name\":\"Unreliability.\",\"subdomain\":\"obakeng\",\"custom_domain\":null,\"custom_domain_optional\":false,\"hero_text\":\"Here we discuss reliability engineering and create experiments to prove ourselves right (or wrong).\",\"logo_url\":null,\"author_id\":75075413,\"primary_user_id\":75075413,\"theme_var_background_pop\":\"#786CFF\",\"created_at\":\"2022-02-04T14:22:04.122Z\",\"email_from_name\":null,\"copyright\":\"Obakeng\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"newspaper\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"is_guest\":false,\"bestseller_tier\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":6,\"comment_count\":0,\"child_comment_count\":0,\"audio_items\":[{\"post_id\":153407591,\"voice_id\":\"en-US-CoraMultilingualNeural\",\"audio_url\":\"https://substack-video.s3.amazonaws.com/video_upload/post/153407591/tts/c2020574-1c1b-4ac0-bd52-c89cb2e67f33/en-US-CoraMultilingualNeural.mp3\",\"type\":\"tts\",\"status\":\"completed\"}],\"is_geoblocked\":false,\"hasCashtag\":false},\"comments\":[],\"canonicalUrl\":\"https://obakeng.substack.com/p/its-a-log-eat-log-world\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://obakeng.substack.com/p/its-a-log-eat-log-world\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":\"#e11d48\",\"background_pop\":\"#e11d48\",\"color_theme_bg_web\":null,\"cover_bg_color\":\"#FFFFFF\",\"cover_bg_color_secondary\":\"#f0f0f0\",\"background_pop_darken\":\"#ca1a41\",\"print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_darken\":\"#ca1a41\",\"color_theme_print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_20\":\"rgba(225, 29, 72, 0.2)\",\"color_theme_bg_pop_30\":\"rgba(225, 29, 72, 0.3)\",\"print_pop\":\"#e11d48\",\"color_theme_accent\":\"#e11d48\",\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#e11d48\",\"font_family_headings_preset\":\"'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'\",\"font_weight_headings_preset\":700,\"font_family_body_preset\":\"'Roboto Mono',monospace\",\"font_weight_body_preset\":400,\"font_size_body_offset\":\"-1px\",\"font_preset_heading\":\"sans\",\"font_preset_body\":\"mono\",\"home_hero\":\"feature-media\",\"home_posts\":\"list\",\"home_show_top_posts\":true,\"web_bg_color\":\"#ffffff\",\"background_contrast_1\":\"#f0f0f0\",\"background_contrast_2\":\"#dddddd\",\"background_contrast_3\":\"#b7b7b7\",\"background_contrast_4\":\"#929292\",\"background_contrast_5\":\"#515151\",\"color_theme_bg_contrast_1\":\"#f0f0f0\",\"color_theme_bg_contrast_2\":\"#dddddd\",\"color_theme_bg_contrast_3\":\"#b7b7b7\",\"color_theme_bg_contrast_4\":\"#929292\",\"color_theme_bg_contrast_5\":\"#515151\",\"color_theme_bg_elevated\":\"#ffffff\",\"color_theme_bg_elevated_secondary\":\"#f0f0f0\",\"color_theme_bg_elevated_tertiary\":\"#dddddd\",\"color_theme_detail\":\"#e6e6e6\",\"background_contrast_pop\":\"rgba(225, 29, 72, 0.4)\",\"color_theme_bg_contrast_pop\":\"rgba(225, 29, 72, 0.4)\",\"theme_bg_is_dark\":\"0\",\"print_on_web_bg_color\":\"#363737\",\"print_secondary_on_web_bg_color\":\"#868787\",\"background_pop_rgb\":\"225, 29, 72\",\"color_theme_bg_pop_rgb\":\"225, 29, 72\",\"color_theme_accent_rgb\":\"225, 29, 72\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":null,\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"125.0\",\"major\":\"125\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
<script nonce="XYBzbX0dMnqLivGDUHjKMw">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"obakeng\",\"publication_id\":730597,\"has_plans\":false,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.7371871e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.571db0c6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.a0731d37.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.d196682b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.2686bb5d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.ae8857e4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.4614ce58.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.41564edd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.b2140fcd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.f3dc2b43.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.7aa24e85.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.1820773d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.765aca5a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.2f1bdde5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.4e726ad5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.f03f24de.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.f579d62b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.80c75c78.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.92b9fc9c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.258e49f2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.d493457b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.ce61e623.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.015b0f71.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.f1207089.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.b91ab2ef.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.b93f6845.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89356.66161d1e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.211a7337.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47925.24a01d36.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.03bd4c41.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71505.3dc43e33.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.5d148f2c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.86d95e8c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.a5a682c3.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/86118.ff1ec843.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.7fe128ac.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.c78aabc1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.d6e9f8ec.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/29231.a7af5911.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.2b32bbb6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.f1c17ded.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.1d0af2b9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.951c11cb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.58e078b0.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.dfde36c8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.94c6b3f1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.a327e9f4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.8bb0d59c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.8f5c5991.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.cb326b80.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.680833b5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.df6a856d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.cf3bcc2c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.8a1bc5dd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.3f20fb36.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.bdbc4e21.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58483.3adddb8a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.0a97506f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.09a9082f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.7f0face7.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/92610.a4fc1daf.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.cb10b4b8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11244.b723f409.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99260.28b0ed6a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.7d5c5ba3.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/76768.9a2f2bae.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/41932.6c688f22.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12114.979fb38e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.9732c6f9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.4f5ae3be.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53871.2923c5c9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.588a06d1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99068.bd2e1fba.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.79d3f415.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.c7f428fb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.098bcbcf.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.2eb51db5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.72d1d844.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.bbf1c283.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.2e8ed950.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/32552.889dd584.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.c287f2d8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.6c265246.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.9b9761ae.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.b40bf629.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.07cd629b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.35b48b09.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.f37fb930.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.24075434.js" charset="utf-8"></script>
<script nonce="XYBzbX0dMnqLivGDUHjKMw" nomodule>
(function() {
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
var warningDiv = document.createElement('div');
warningDiv.style.color = 'red';
warningDiv.style.padding = '10px';
warningDiv.style.margin = '10px 0';
warningDiv.style.border = '1px solid red';
warningDiv.style.backgroundColor = 'lightyellow';
warningDiv.innerText = message;
document.body.prepend(warningDiv);
})();
</script>
<!-- Datadog Analytics -->
<script nonce="XYBzbX0dMnqLivGDUHjKMw">
(function(h,o,u,n,d) {
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
d=o.createElement(u);d.async=1;d.src=n
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
window.DD_RUM.onReady(function() {
window.DD_RUM.init({
clientToken: 'puba71073f072643721169b68f352438710',
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
site: 'datadoghq.com',
service: 'web',
env: window._preloads.dd_env,
version: '8b4c00716b2bc0f7ca1fded553014f7e37b5b07a',
sessionSampleRate: 1,
sessionReplaySampleRate: 10,
trackUserInteractions: window._preloads.dd_ti,
trackResources: true,
trackLongTasks: true,
defaultPrivacyLevel: 'mask-user-input',
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
});
})
</script>
<!-- End Datadog Analytics -->
<!-- Cloudflare Web Analytics -->
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
<!-- End Cloudflare Web Analytics -->
<!-- Fallback tracking pixels -->
<noscript>
<style>
#nojs-banner {
position: fixed;
bottom: 0;
left: 0;
padding: 16px 16px 16px 32px;
width: 100%;
box-sizing: border-box;
background: red;
color: white;
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
font-size: 13px;
line-height: 13px;
}
#nojs-banner a {
color: inherit;
text-decoration: underline;
}
</style>
<div id="nojs-banner">
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
</div>
</noscript>
<script nonce="XYBzbX0dMnqLivGDUHjKMw">(function(){function c(){var b=a.contentDocument||(a.contentWindow&&a.contentWindow.document);if(b){var d=b.createElement('script');d.nonce='XYBzbX0dMnqLivGDUHjKMw';d.innerHTML="window.__CF$cv$params={r:'a39ce7390be0885c',t:'MTc4OTE5NTA0Mg=='};var a=document.createElement('script');a.nonce='XYBzbX0dMnqLivGDUHjKMw';a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();</script></body>
</html>