508 lines
242 KiB
HTML
508 lines
242 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en" dir="ltr">
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
|
||
|
||
|
||
<link rel="preconnect" href="https://substackcdn.com" />
|
||
|
||
|
||
|
||
<title data-rh="true">Live Your Best Life With Structured Events</title>
|
||
|
||
<meta data-rh="true" name="theme-color" content="#ffffff"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Live Your Best Life With Structured Events"/><meta data-rh="true" name="twitter:title" content="Live Your Best Life With Structured Events"/><meta data-rh="true" name="description" content="If you’re like most of us, you learned to debug as a baby engineer by way of printf(3)."/><meta data-rh="true" property="og:description" content="If you’re like most of us, you learned to debug as a baby engineer by way of printf(3)."/><meta data-rh="true" name="twitter:description" content="If you’re like most of us, you learned to debug as a baby engineer by way of printf(3)."/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!khu1!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fcharitydotwtf.substack.com%2Ftwitter%2Fsubscribe-card.jpg%3Fv%3D-281824914%26version%3D9"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!khu1!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fcharitydotwtf.substack.com%2Ftwitter%2Fsubscribe-card.jpg%3Fv%3D-281824914%26version%3D9"/><meta data-rh="true" name="twitter:card" content="summary"/><meta data-rh="true" property="article:modified_time" content="2025-12-16T00:58:55.877Z"/>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@layer legacy, pencraftReset, pencraft;
|
||
</style>
|
||
|
||
|
||
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
|
||
|
||
|
||
|
||
<link rel="preload" as="font" href="https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M5knjsS_ul.woff2" crossorigin />
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.cee9fbee.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/86118.c25b9f67.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/76768.f15fc605.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/41932.97d2663c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/32552.313415d5.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
|
||
<meta name="author" content="Charity Majors" />
|
||
<meta property="og:url" content="https://charity.wtf/p/live-your-best-life-with-structured-events" />
|
||
|
||
|
||
<link rel="canonical" href="https://charity.wtf/p/live-your-best-life-with-structured-events" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="shortcut icon" href="https://substackcdn.com/image/fetch/$s_!jvlR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Ffavicon.ico">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="16x16" href="https://substackcdn.com/image/fetch/$s_!N42a!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Ffavicon-16x16.png">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="32x32" href="https://substackcdn.com/image/fetch/$s_!hIAJ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Ffavicon-32x32.png">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="48x48" href="https://substackcdn.com/image/fetch/$s_!-tuJ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Ffavicon-48x48.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="57x57" href="https://substackcdn.com/image/fetch/$s_!_ZAb!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-57x57.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="60x60" href="https://substackcdn.com/image/fetch/$s_!garv!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-60x60.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="72x72" href="https://substackcdn.com/image/fetch/$s_!YuQM!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-72x72.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="76x76" href="https://substackcdn.com/image/fetch/$s_!vaZL!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-76x76.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="114x114" href="https://substackcdn.com/image/fetch/$s_!Hvv-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-114x114.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="120x120" href="https://substackcdn.com/image/fetch/$s_!DfUU!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-120x120.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="144x144" href="https://substackcdn.com/image/fetch/$s_!TIIj!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-144x144.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="152x152" href="https://substackcdn.com/image/fetch/$s_!Genc!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-152x152.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="167x167" href="https://substackcdn.com/image/fetch/$s_!3nCK!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-167x167.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="180x180" href="https://substackcdn.com/image/fetch/$s_!I9xu!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-180x180.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="1024x1024" href="https://substackcdn.com/image/fetch/$s_!R7My!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-1024x1024.png">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="alternate" type="application/rss+xml" href="/feed" title="charity.wtf"/>
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8on7mTNmnUHowCw.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onXmTNmnUHowCw.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onTmTNmnUHowCw.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onrmTNmnUHo.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M9knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M2knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M3knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M5knjsS_ul.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3FafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3OafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3PafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3BafaPWnII.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||
|
||
</style>
|
||
|
||
|
||
|
||
|
||
|
||
<style data-pub-theme-prefix>:root{--background_pop:#ff6719;--cover_bg_color:#FFFFFF;--cover_bg_color_secondary:#f0f0f0;--background_pop_darken:#ff5600;--print_on_pop:#ffffff;--print_pop:#ff6719;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#ff6719;--background_contrast_1:#f0f0f0;--background_contrast_2:#dddddd;--background_contrast_3:#b7b7b7;--background_contrast_4:#929292;--background_contrast_5:#515151;--color_theme_detail:#e6e6e6;--background_contrast_pop:rgba(255, 103, 25, 0.4);--theme_bg_is_dark:0;--background_pop_rgb:255, 103, 25;}:root:where(:has(> body:not(.app-theme))){--color_theme_bg_pop:#FF6719;--background_pop:#FF6719;--cover_bg_color:#FFFFFF;--cover_bg_color_secondary:#f0f0f0;--background_pop_darken:#ff5600;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#ff5600;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(255, 103, 25, 0.2);--color_theme_bg_pop_30:rgba(255, 103, 25, 0.3);--print_pop:#ff6719;--color_theme_accent:#ff6719;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#ff6719;--home_hero:newspaper;--home_posts:list;--web_bg_color:#ffffff;--background_contrast_1:#f0f0f0;--background_contrast_2:#dddddd;--background_contrast_3:#b7b7b7;--background_contrast_4:#929292;--background_contrast_5:#515151;--color_theme_bg_contrast_1:#f0f0f0;--color_theme_bg_contrast_2:#dddddd;--color_theme_bg_contrast_3:#b7b7b7;--color_theme_bg_contrast_4:#929292;--color_theme_bg_contrast_5:#515151;--color_theme_bg_elevated:#ffffff;--color_theme_bg_elevated_secondary:#f0f0f0;--color_theme_bg_elevated_tertiary:#dddddd;--color_theme_detail:#e6e6e6;--background_contrast_pop:rgba(255, 103, 25, 0.4);--color_theme_bg_contrast_pop:rgba(255, 103, 25, 0.4);--theme_bg_is_dark:0;--print_on_web_bg_color:#363737;--print_secondary_on_web_bg_color:#868787;--background_pop_rgb:255, 103, 25;--color_theme_bg_pop_rgb:255, 103, 25;--color_theme_accent_rgb:255, 103, 25;}</style>
|
||
|
||
|
||
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
|
||
|
||
|
||
<style data-pub-theme-postfix></style>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</head>
|
||
|
||
<body class="">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div id="entry">
|
||
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:71px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="flex-basis:0px;flex-grow:1;" class="logoContainer-p12gJb"><div class="pencraft pc-display-flex pc-gap-40 pc-alignItems-center pc-reset"><a tabindex="-1" href="https://substack.com?utm_content=island-substack-logo" class="pencraft pc-display-flex pc-width-20 pc-height-20 pc-justifyContent-center pc-alignItems-center pc-reset"><svg aria-hidden="true" style="transform:scale(1.4);" width="16" height="18" viewBox="0 0 16 18" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M16 4H0V6H16V4Z"></path><path d="M0 8V18L7.9993 13.534L16 18V8H0Z"></path><path d="M16 0H0V2H16V0Z"></path></g></svg></a></div></div><div style="flex-grow:0;" class="titleContainer-DJYq5v"><h1 class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd"><a href="/" class="pencraft pc-display-contents pc-reset">Charity Majors</a></h1></div><div style="flex-basis:0px;flex-grow:1;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Flive-your-best-life-with-structured-events&for_pub=charitydotwtf" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="height:72px;"></div></div></div><div><script nonce="HnqQfs3nO8pxxyyQ577RVQ" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://charity.wtf/p/live-your-best-life-with-structured-events","mainEntityOfPage":"https://charity.wtf/p/live-your-best-life-with-structured-events","headline":"Live Your Best Life With Structured Events","description":"If you\u2019re like most of us, you learned to debug as a baby engineer by way of printf(3).","image":[],"datePublished":"2022-08-15T06:32:29+00:00","dateModified":"2022-08-15T06:32:29+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"Charity Majors","url":"https://substack.com/@charitydotwtf","description":"cofounder and CTO of honeycomb.io; pioneered modern observability. co-author of O'Reilly books \"Database Reliability Engineering\" and \"Observability Engineering\", now wrapping up the 2nd ed. loves free software, free speech, and peaty single malts.","identifier":"user:32306597","image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!EAp-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!EAp-!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg"}}],"publisher":{"@type":"Organization","name":"charity.wtf","url":"https://charity.wtf","description":"observability, tech advice, honeycomb.io, etc","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":1000},"identifier":"pub:2935724","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!Tg4H!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!Tg4H!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!Tg4H!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!Tg4H!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!Tg4H!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!Tg4H!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png"}},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":0},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":0},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":0}]}</script><script nonce="HnqQfs3nO8pxxyyQ577RVQ" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"charity.wtf","item":"https://charity.wtf"},{"@type":"ListItem","position":2,"name":"Live Your Best Life With Structured Events","item":"https://charity.wtf/p/live-your-best-life-with-structured-events"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><h1 dir="auto" class="post-title published title-X77sOw">Live Your Best Life With Structured Events</h1><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-row pc-gap-8 pc-alignItems-center pc-justifyContent-flex-start pc-reset"><div style="--scale:36px;--offset:9px;--border-width:4.5px;" class="pencraft pc-display-flex pc-flexDirection-row pc-alignItems-center pc-justifyContent-flex-start pc-reset ltr-qDBmby"><a href="https://substack.com/@charitydotwtf" aria-label="View Charity Majors's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:36px;" tabindex="0" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6 avatar-u8q6xB last-JfNEJ_"><div style="--scale:36px;" title="Charity Majors" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!EAp-!,w_36,h_36,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 1x, https://substackcdn.com/image/fetch/$s_!EAp-!,w_72,h_72,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 2x, https://substackcdn.com/image/fetch/$s_!EAp-!,w_108,h_108,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 3x"/><img src="https://substackcdn.com/image/fetch/$s_!EAp-!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg" alt="Charity Majors's avatar" srcset="https://substackcdn.com/image/fetch/$s_!EAp-!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 1x, https://substackcdn.com/image/fetch/$s_!EAp-!,w_72,h_72,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 2x, https://substackcdn.com/image/fetch/$s_!EAp-!,w_108,h_108,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 3x" width="36" height="36" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-reset color-primary-zABazT line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><span style="min-width:0;" data-state="closed"><a href="https://substack.com/@charitydotwtf" class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ">Charity Majors</a></span></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2022-08-15T06:32:29.000Z" title="2022-08-15T06:32:29.000Z">Aug 15, 2022</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://charity.wtf/p/live-your-best-life-with-structured-events/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-27" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p><span>If you’re like most of us, you learned to debug as a baby engineer by way of printf(3). By the time you were shipping code to production you had probably learned to instrument your code with a real metrics library. Maybe a tenth of us learned to use gdb and still step through functions on the regular. (I said </span><em>maybe</em><span>.)</span></p><p><span>Printing stuff to stdout is still the Swiss Army knife of tools. Always there when you reach for it, usually helps more than it does harm. (I said </span><em>usually.</em><span>)</span></p><p><span>And then! In case you’ve been living under a rock, we recently went and blew up ye aulde monolythe, and in the process we … lost most of our single-process tools and techniques for debugging. Forget gdb; even </span><em>printf</em><span> doesn’t work when you’re hopping the network between functions.</span></p><p><span>If your tool set no longer works for you, friend, it’s time to go all in. Maybe what you </span><em>wanted</em><span> was a faster horse, but it’s time for a car, and the sooner you turn in your oats for gas cans and a spare tire, the better.</span></p><h2 class="header-anchor-post">Exercising Good Technical Judgment (When You Don’t Have Any)<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§exercising-good-technical-judgment-when-you-dont-have-any" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/exercising-good-technical-judgment-when-you-dont-have-any" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>If you’re stuck trying to debug modern problems with pre-modern tooling, the first thing to do is stop digging the hole. Stop pushing good data after bad into formats and stores that aren’t going to help you answer the right questions.</p><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!ApgF!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!ApgF!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 424w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 848w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!ApgF!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg" width="236" height="218" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/d3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":218,"width":236,"resizeWidth":null,"bytes":null,"alt":"0893d048d8361fe632b090b0429ad78b-rainbow-dash-rainbows-e1542789580565.jpg","title":null,"type":null,"href":null,"belowTheFold":false,"topImage":true,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt="0893d048d8361fe632b090b0429ad78b-rainbow-dash-rainbows-e1542789580565.jpg" title="0893d048d8361fe632b090b0429ad78b-rainbow-dash-rainbows-e1542789580565.jpg" srcset="https://substackcdn.com/image/fetch/$s_!ApgF!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 424w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 848w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 1456w" sizes="100vw" fetchpriority="high" class="sizing-normal"/></picture><div></div></div></a><p><span>In brief: if you aren’t rolling out a solution based on </span><strong>arbitrarily wide, structured raw events that are unique and ordered and trace-aware </strong><span>and without any aggregation at write time, you are going to regret it. (If you aren’t using OpenTelemetry, you are going to regret that, too.)</span></p><p>So just make the leap as soon as possible.</p><p>But let’s rewind a bit. Let’s start with observability.</p><h2 class="header-anchor-post">Observability: an introduction<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§observability-an-introduction" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/observability-an-introduction" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Observability is not a new word or concept, but the definition of observability as a specific technical term applied to software engineering is relatively new — about four years old. Before that, if you heard the term in softwareland it was only as a generic synonym for telemetry (“there are three pillars of observability”, in one annoying formulation) or team names (twitter, for example, has long had an “observability team”).</p><p><span>The term itself originates with </span><a href="https://en.wikipedia.org/wiki/Observability">control theory:</a></p><p>“In control theory, observability is a measure of how well internal states of a system can be inferred from knowledge of its external outputs. The observability and controllability of a system are mathematical duals. The concept of observability was introduced by Hungarian-American engineer Rudolf E. Kálmán for linear dynamic systems.[1][2]”</p><p><span>But when applied to a software context, observability refers to how well you can understand and reason about your systems, just by interrogating them and inspecting their outputs with your tools. How well can you understand the </span><em>inside</em><span> of the system from the </span><em>outside</em><span>?</span></p><p><span>Achieving this relies your ability to ask brand new questions, questions you have never encountered and never anticipated — </span><em><strong>without shipping new code</strong><span>. </span></em><span>Shipping new code is cheating, because it means that you knew in advance what the problem was in order to instrument it.</span></p><h3 class="header-anchor-post">But what about monitoring?<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§but-what-about-monitoring" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/but-what-about-monitoring" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>Monitoring has a long and robust history, but it has always been about watching your systems for failures you can define and expect. Monitoring is for known-unknowns, and setting thresholds and running checks against the system. </span><strong>Observability is about the</strong></p><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!RTFM!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!RTFM!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 424w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 848w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!RTFM!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg" width="300" height="169" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":169,"width":300,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt title srcset="https://substackcdn.com/image/fetch/$s_!RTFM!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 424w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 848w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><p><strong> unknown-unknowns.</strong><span> Which requires a fundamentally different mindset and toolchain.</span></p><p><span>“Monitoring is the action of observing and checking the behavior and outputs of a system and its components over time.” — @grepory, in his talk “</span><a href="https://speakerdeck.com/grepory/monitoring-is-dead">Monitoring is Dead</a><span>“.</span></p><p><strong>Monitoring is a third-person perspective on your software</strong><span>. It’s not software explaining itself from the inside out, it’s one piece of software checking up on another.</span></p><h3 class="header-anchor-post">Observability is for understanding complex, ephemeral, dynamic systems (not for debugging code)<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§observability-is-for-understanding-complex-ephemeral-dynamic-systems-not-for-debugging-code" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/observability-is-for-understanding-complex-ephemeral-dynamic-systems-not-for-debugging-code" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>You don’t use observability for stepping through functions; it’s not a debugger. Observability is for swiftly identifying where in your system the error or problem is coming from, so you can debug it — by reproducing it, or seeing what it has in common with other erroring requests. You can think of observability as being like B.I. (business intelligence) tooling for software applications, in the way you engage in a lot of exploratory, open-ended data sifting to detect novel patterns and behaviors.</p><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!rm9f!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!rm9f!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 424w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 848w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 1272w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!rm9f!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png" width="242" height="220" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/f32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":220,"width":242,"resizeWidth":null,"bytes":null,"alt":"rainbow_dash___no_by_cptofthefriendship-d4erd69","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt="rainbow_dash___no_by_cptofthefriendship-d4erd69" title="rainbow_dash___no_by_cptofthefriendship-d4erd69" srcset="https://substackcdn.com/image/fetch/$s_!rm9f!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 424w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 848w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 1272w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><p><span>Observability is often about swiftly isolating or tracking down the problem in your large, sprawling, far-flung, dynamic system. Because the hard part of distributed systems is rarely debugging the code, </span><strong>it’s figuring out where the code you need to debug is</strong><span>.</span></p><p>The need for observability is often associated with microservices adoption, because they are prohibitively difficult to debug without service-level event oriented tooling — the kind you can get from Honeycomb and Lightstep.. and soon, I hope, many other vendors.</p><h2 class="header-anchor-post">Events are the building blocks of observability<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§events-are-the-building-blocks-of-observability" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/events-are-the-building-blocks-of-observability" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Ergh, another overloaded data term. What even is an “event”?</p><p><span>An observability “event” is a hop in the lifecycle of an end-to-end request. If a request executes code on three services separated by network hops before returning to the user, that request generated three observability “events”, each packed with context and details about that code running in that environment. These are also sometimes called “</span><a href="https://stripe.com/blog/canonical-log-lines">canonical log lines</a><span>“. If you implemented tracing, each event may be a span in your trace.</span></p><p>If request ID #A897BEDC hits your edge, then your API service, then four more internal services, and twice connects to a db and runs a query, then request ID #A897BEDC generated 8 observability events … assuming you are in fact gathering observability data from the edge, the API, the internal services and the databases.</p><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!hhTU!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!hhTU!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 424w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 848w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 1272w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!hhTU!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png" width="176" height="167" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/f42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":167,"width":176,"resizeWidth":null,"bytes":null,"alt":"ponyfm-i7812-original","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt="ponyfm-i7812-original" title="ponyfm-i7812-original" srcset="https://substackcdn.com/image/fetch/$s_!hhTU!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 424w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 848w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 1272w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><p><span>This is an important caveat. </span><strong>We only gather observability events from services that we can and do introspect</strong><span>. If it’s a black box to us, that hop cannot generate an observability event. So if request ID #A897BEDC also performed 20 cache lookups and called out to 8 external HTTP services and 2 managed databases, those 30 hops do </span><em>not</em><span> generate observability events (assuming you haven’t instrumented the memcache service and have no instrumentation from those external services/dbs). Each request generates one event per request per service hop.**</span></p><p><span>(I also wrote about </span><a href="https://charity.wtf/2019/02/05/logs-vs-structured-events/">logs vs structured events here</a><span>.)</span></p><h2 class="header-anchor-post"><strong>Observability is a first-person narrative</strong><span>.</span><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§observability-is-a-first-person-narrative" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/observability-is-a-first-person-narrative" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>We care primarily about self-reported status from the code as it executes the request path.</p><h2 class="header-anchor-post"><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!k_6J!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!k_6J!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 424w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 848w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 1272w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!k_6J!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png" width="300" height="186" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/d2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":186,"width":300,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt title srcset="https://substackcdn.com/image/fetch/$s_!k_6J!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 424w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 848w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 1272w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><p>Instrumentation is your eyes and ears, explaining the software and its environment from the perspective of your code. Monitoring, on the other hand, is traditionally a third-person narrative — it’s one piece of software checking up on another piece of software, with no internal knowledge of its hopes and dreams.</p><p><span>First-person narrative reports have the best potential for telling a reliable narrative. And more importantly, they </span><strong>map directly to user experience</strong><span> in a way that third-party monitoring does not and cannot.</span></p><h3 class="header-anchor-post">Events … must be structured.<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§events-must-be-structured" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/events-must-be-structured" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>First, </span><strong>structure your goddamn data</strong><span>. You’re a computer scientist, you’ve got no business using text search to plow through terabytes of text.</span></p><h3 class="header-anchor-post">Events … are not just structured logs.<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§events-are-not-just-structured-logs" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/events-are-not-just-structured-logs" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>Now, part of the reason people seem to think structured data is cost-prohibitive is that </span><strong>they’re doing it wrong</strong><span>. They’re still thinking about these like log lines. And while you </span><em>can</em><span> look at events like they’re just really wide structured log lines that aren’t flushed to disk, here’s why you shouldn’t: logs have decades of abhorrent associations and absolutely ghastly practices.</span></p><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!AFeN!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!AFeN!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 424w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 848w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 1272w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!AFeN!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png" width="170" height="171" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/e44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":171,"width":170,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt title srcset="https://substackcdn.com/image/fetch/$s_!AFeN!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 424w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 848w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 1272w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><p>Instead of bundling up and passing along one neat little pile of context, they’re spewing log lines inside loops in their code and DDoS’ing their own logging clusters.They’re shitting out “log lines” with hardly any dimensions so they’re information-sparse and just straight up wasting the writes. And then to compensate for the sparseness and repetitiveness they just start logging the same exact nouns tens or hundreds of times over the course of the request, just so they can correlate or reconstruct some lousy request that they never should have blown up in the first place!</p><p>But they keep hearing they should be structuring their logs, so they pile structure on to their horrendous little strings, which pads every log line by a few bytes, so their bill goes up but they aren’t getting any benefit! just paying more! What the hell, structuring is bull shit!</p><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!PZ-U!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!PZ-U!,w_424,c_limit,f_webp,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 424w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_848,c_limit,f_webp,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 848w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_1272,c_limit,f_webp,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 1272w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_1456,c_limit,f_webp,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!PZ-U!,w_1456,c_limit,f_auto,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif" width="320" height="320" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/e692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":196,"width":196,"resizeWidth":null,"bytes":null,"alt":"giphy","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt="giphy" title="giphy" srcset="https://substackcdn.com/image/fetch/$s_!PZ-U!,w_424,c_limit,f_auto,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 424w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_848,c_limit,f_auto,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 848w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_1272,c_limit,f_auto,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 1272w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_1456,c_limit,f_auto,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><p>Kittens. You need a fundamentally different approach to reap the considerable benefits of structuring your data.</p><p>But the difference between strings and structured data is ~basically the difference between grep and all of computer science. 😛</p><h3 class="header-anchor-post">Events … must be arbitrarily wide and dense with context.<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§events-must-be-arbitrarily-wide-and-dense-with-context" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/events-must-be-arbitrarily-wide-and-dense-with-context" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>So the most effective way to structure your instrumentation, to get the absolute most bang for your buck, is to emit a single arbitrarily wide event per request per service hop. At Honeycomb, the maturely instrumented datasets that we see are often 200-500 dimensions wide. Here’s an event that’s just 20 dimensions wide:</p><pre><code>{
|
||
|
||
"timestamp":"2018-11-20 19:11:56.910",
|
||
"az":"us-west-1",
|
||
"build_id":"3150",
|
||
"customer_id":"2310",
|
||
"durationMs":167,
|
||
"endpoint":"/api/v2/search",
|
||
"endpoint_shape":"/api/v2/search",
|
||
"fraud_dur":131,
|
||
"hostname":"app14",
|
||
"id":"f46691dfeda9ede4",
|
||
"mysql_dur":"",
|
||
"name":"/api/v2/search",
|
||
"parent_id":"",
|
||
"platform":"android",
|
||
"query":"",
|
||
"serviceName":"api",
|
||
"status_code":200,
|
||
"traceId":"f46691dfeda9ede4",
|
||
"user_id":"344310",
|
||
"error_rate":0,
|
||
"is_root":"true"
|
||
}</code></pre><p><span>So a well-instrumented service should have hundreds of these dimensions, all bundled around the context of each request. And yet — and here’s why events blow the pants off of metrics — even with hundreds of dimensions, it’s still </span><em>just one write</em><span>. </span><strong>Adding more dimensions to your event is effectively free,</strong><span> it’s still one write plus a few more bits.</span></p><p>Compare this to a metric-based systems, where you are often in the position of trying to predict whether a metric will be valuable enough to justify the extra write, because every single metric or tag you add contributes linearly to write amplification. Ever gotten billed tens of thousands of dollars for your custom metrics, or had to prune your list of useful custom metrics down to something affordable? (“BUT THOSE ARE THE ONLY USEFUL ONES!”, as every ops team wails)</p><h3 class="header-anchor-post"><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!o6VP!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!o6VP!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 424w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 848w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 1272w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!o6VP!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png" width="300" height="218" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/f1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":218,"width":300,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt title srcset="https://substackcdn.com/image/fetch/$s_!o6VP!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 424w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 848w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 1272w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><h3 class="header-anchor-post">Events … must pass along the blob of context as the request executes<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§events-must-pass-along-the-blob-of-context-as-the-request-executes" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/events-must-pass-along-the-blob-of-context-as-the-request-executes" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>As you can imagine, it can be a pain in the ass to keep passing this blob of information along the life of the request as it hits many services and databases. So at Honeycomb we do all the annoying parts for you with our integrations. You just install the go pkg or ruby gem or whatever, and under the hood we:</p><ol><li><p>initialize an empty debug event when the request enters that service</p></li><li><p>prepopulate the empty debug event with any and all interesting information that we already know or can guess. language type, version, environment, etc.</p></li><li><p>create a framework so you can just stuff any other details in there as easily as if you were printing it out to stdout</p></li><li><p>pass the event along and maintain its state until you are ready to error or exit</p></li><li><p>write the extremely wide event out to honeycomb</p></li></ol><p>Easy!</p><p><span>(Check out this killer talk from </span><a href="http://twitter.com/lyddonb">@lyddonb</a><span> on … well everything you need to know about life, love and distributed systems is in here, but around the 12:00 mark he describes why this approach is mandatory. WATCH IT. </span><a href="https://www.youtube.com/watch?v=xy3w2hGijhE&feature=youtu.be">https://www.youtube.com/watch?v=xy3w2hGijhE&feature=youtu.be</a><span>)</span></p><h3 class="header-anchor-post">Events … should collect context like sticky buns collect dust<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§events-should-collect-context-like-sticky-buns-collect-dust" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/events-should-collect-context-like-sticky-buns-collect-dust" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Other stuff you’ll want to track in these structured blobs includes:</p><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!LfJi!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!LfJi!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 424w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 848w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 1272w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!LfJi!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png" width="190" height="221" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":221,"width":190,"resizeWidth":null,"bytes":null,"alt":"1225287_1370081029072_full","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt="1225287_1370081029072_full" title="1225287_1370081029072_full" srcset="https://substackcdn.com/image/fetch/$s_!LfJi!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 424w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 848w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 1272w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><ol><li><p>Metadata like src, dst headers</p></li><li><p>The timing stats and contents of every network call (our beelines wrap all outgoing http calls and db queries automatically)</p></li><li><p>Every raw db query, normalized query family, execution time etc</p></li><li><p>Infra details like AZ, instance type, provider</p></li><li><p>Language/environment context like $lang version, build flags, $ENV variables</p></li><li><p><span>Any and all unique identifying bits you can get your grubby little paws on — request ID, shopping cart ID, user ID, request ID, transaction ID, any other ID … </span><strong>these are always the highest value data</strong><span> for debugging.</span></p></li><li><p>Any other useful application context. Service name, build id, ordering info, error rates, cache hit rate, counters, whatever.</p></li><li><p>Possibly the system resource state at this point in time. e.g. values from /proc/net/ipv4 stats</p></li></ol><p><span>Capture all of it. Anything that ever occurs to you </span><em>(“this MIGHT be handy someday”) — </em><span>don’t even hesitate, just throw it on the pile. Collect it up in one rich fat structured blob.</span></p><h3 class="header-anchor-post">Events … must be unique, ordered, and traceable<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§events-must-be-unique-ordered-and-traceable" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/events-must-be-unique-ordered-and-traceable" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>You need a unique request ID, and you need to propagate it through your stack in some way that preserves sequence. Once you have that, traces are just a beautiful visualization layer on top of your shiny event data.</p><h3 class="header-anchor-post">Events … must be stored raw.<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§events-must-be-stored-raw" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/events-must-be-stored-raw" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Because observability means you need to be able to ask any arbitrary new question of</p><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!F7aC!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!F7aC!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 424w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 848w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 1272w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!F7aC!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png" width="181" height="125" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":125,"width":181,"resizeWidth":null,"bytes":null,"alt":"Rainbow-Dash-is-not-amused-my-little-pony-friendship-is-magic-31088082-900-622","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt="Rainbow-Dash-is-not-amused-my-little-pony-friendship-is-magic-31088082-900-622" title="Rainbow-Dash-is-not-amused-my-little-pony-friendship-is-magic-31088082-900-622" srcset="https://substackcdn.com/image/fetch/$s_!F7aC!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 424w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 848w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 1272w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><p>your system without shipping new code, and aggregation is a one-way trip. Once you have aggregated your data and discarded the raw requests, you have destroyed your ability to ask new questions of that data forever. For Ever.</p><p><strong>Aggregation is a one-way trip</strong><span>. You can always, always derive your pretty metrics and dashboards and aggregates from structured events, and you can never go in reverse. Same for traces, same for logs. The structured event is the gold standard. Invest in it now, save your ass in the future.</span></p><p>It’s only observability if you can ask new questions. And that means storing raw events.</p><h3 class="header-anchor-post">Events…are richer than metrics<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§eventsare-richer-than-metrics" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/eventsare-richer-than-metrics" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>There’s always tradeoffs when it comes to data. Metrics choose to sacrifice context and connective tissue, and sometimes high cardinality support, which you need to correlate anomalies or track down outliers. They have a very small, efficient data format, but they sacrifice everything else by discarding all but the counter, gauge, etc.</p><p>A metric looks like this, by the way.</p><pre><code>{ metric: "db.query.time", value: 0.502, tags: Array(), type: set }</code></pre><p>That’s it. It’s just a name, a number and maybe some tags. You can’t dig into the event and see what else was happening when that query was strangely slow. You can never get that information back after discarding it at write time.</p><p>But because they’re so cheap, you can keep every metric for every request! Maybe. (Sometimes.) More often, what happens is they aggregate at write time. So you never actually get a value written out for an individual event, it smushes everything together that happens in the 1 second interval and calculates some aggregate values to write out. And that’s all you can ever get back to.</p><p>With events, and their relative explosion of richness, we sacrifice our ability to store every single observability event about every request. At FB, every request generated hundreds of observability events as it made its way through the stack. Nobody, NOBODY is going to pay for an o11y stack that is hundreds of times as large as production. The solution to that problem is sampling.</p><h3 class="header-anchor-post">Events…should be sampled.<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§eventsshould-be-sampled" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/eventsshould-be-sampled" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!M0ok!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!M0ok!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 424w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 848w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 1272w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!M0ok!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png" width="242" height="220" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/87abc044-0ee7-413d-af84-819b99916fde_300x273.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":220,"width":242,"resizeWidth":null,"bytes":null,"alt":"rainbow_dash___no_by_cptofthefriendship-d4erd69","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt="rainbow_dash___no_by_cptofthefriendship-d4erd69" title="rainbow_dash___no_by_cptofthefriendship-d4erd69" srcset="https://substackcdn.com/image/fetch/$s_!M0ok!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 424w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 848w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 1272w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><p>But not dumb, blunt sampling at server side. Control it on the client side.</p><p>Then sample heavily for events that are known to be common and useless, but keep the events that have interesting signal. For example: health checks that return 200 OK usually represent a significant chunk of your traffic and are basically useless, while 500s are almost always interesting. So are all requests to /login or /payment endpoints, so keep all of them. For database traffic: SELECTs for health checks are useless, DELETEs and all other mutations are rare but you should keep all of them. Etc.</p><p>You don’t need to treat your observability metadata with the same care as you treat your billing data. That’s just dumb.</p><h2 class="header-anchor-post">… To be continued.<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§to-be-continued" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181745827/to-be-continued" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>I hope it’s now blazingly obvious why observability requires — REQUIRES — that you have access to raw structured events with no pre-aggregation or write-time rollups. Metrics don’t count. Just traces don’t count. Unstructured logs sure as fuck don’t count.</p><p>Structured, arbitrarily wide events, with dynamic sampling of the boring parts to control costs. There is no substitute.</p><p><span>For more about the technical requirements for observability, read </span><a href="https://www.honeycomb.io/blog/so-you-want-to-build-an-observability-tool/">this</a><span>, </span><a href="https://thenewstack.io/observability-a-3-year-retrospective/">this</a><span>, or </span><a href="https://thenewstack.io/observability-the-5-year-retrospective/">this</a><span>.</span></p><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!emOQ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!emOQ!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 424w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 848w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!emOQ!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg" width="195" height="260" data-attrs="{"src":"https://substack-post-media.s3.amazonaws.com/public/images/c2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":260,"width":195,"resizeWidth":null,"bytes":null,"alt":"IMG_4619","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}" alt="IMG_4619" title="IMG_4619" srcset="https://substackcdn.com/image/fetch/$s_!emOQ!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 424w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 848w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a><p><em>**The deep fine print: it’s one observability event per request per service hop … because we gather observability detail organized by request id. Databases may be different. For example, with MongoDB or MySQL, we can’t instrument them to talk to honeycomb directly, so we gather information about its internal perspective by 1) tailing the slow query log (and turning it up to log all queries if perf allows), 2) streaming tcp over the wire and reconstructing transactions, 3) connecting to the mysql port as root every couple seconds from cron, then dumping all mysql stats and streaming them in to honeycomb as an event. SO. Database traffic is not organized around connection length or unique request id, it is organized around transaction id or query id. Therefore it generates one observability event per query or transaction. </em></p><p><em>In other words: if your request hit the edge, API, four internal services, two databases … but ran 1 query on one db and 10 queries on the second db … you would generate a total of *19 observability events* for this request.</em></p><p><span>For more on observability for databases and other black boxes, try</span><a href="https://www.honeycomb.io/blog/add-observability-to-databases/"> this blog post</a><span>.</span></p></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://charity.wtf/p/live-your-best-life-with-structured-events/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-56" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="visibility-check"></div><div id="discussion" class="pencraft pc-display-flex pc-flexDirection-column pc-gap-16 pc-paddingTop-32 pc-paddingBottom-32 pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-32 pc-reset container"><h4 class="pencraft pc-reset line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Discussion about this post</h4><div class="pencraft pc-alignSelf-flex-start pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Select discussion type" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-70-content-comments" data-state="active" id="radix-P0-70-trigger-comments" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Comments</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-70-content-restacks" data-state="inactive" id="radix-P0-70-trigger-restacks" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Restacks</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div></div></div><div id="substack-comments" class="single-post-section comments-section"><div class="container"><div class="visibility-check"></div><div data-test-id="comment-input" class="pencraft pc-display-flex pc-reset flex-grow-rzmknG"><form class="comment-input-form form-CkZ7Kt"><div style="--scale:32px;" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><div style="--scale:32px;" title="User" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-flexDirection-column pc-boxShadow-xs pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-sm"><textarea name="body" placeholder="Write a comment..." aria-label="Write a comment..." rows="4" class="pencraft input-qHk4bN autogrowing-_ipn9Y textarea-GbEjRX inputText-pV_yWb unstyled-IJLUJU"></textarea></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></form></div></div></div></div><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-83-content-top" data-state="active" id="radix-P0-83-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-83-content-new" data-state="inactive" id="radix-P0-83-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-83-content-community" data-state="inactive" id="radix-P0-83-trigger-community" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Discussions</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div style="--color-app-light-accent:rgb(255 103 25 / 1);--color-app-light-accent-hover:rgb(255 86 0 / 1);--color-app-light-accent-inverse:rgb(255 255 255 / 1);--color-app-light-accent-text:rgb(255 103 25 / 1);--color-app-dark-accent:rgb(255 103 25 / 1);--color-app-dark-accent-hover:rgb(255 132 73 / 1);--color-app-dark-accent-inverse:rgb(255 255 255 / 1);--color-app-dark-accent-text:rgb(255 103 25 / 1);--color-app-light-bg-accent:rgb(255 103 25 / 0.2);--color-app-light-bg-accent-hover:rgb(255 103 25 / 0.3);--color-app-dark-bg-accent:rgb(255 103 25 / 0.2);--color-app-dark-bg-accent-hover:rgb(255 103 25 / 0.3);--color-app-light-fg-primary:rgb(54 55 55 / 1);--color-app-light-fg-secondary:rgb(134 135 135 / 1);--color-app-light-fg-tertiary:rgb(146 146 146 / 1);--color-app-dark-fg-primary:rgb(255 255 255 / 0.9);--color-app-dark-fg-secondary:rgb(255 255 255 / 0.6);--color-app-dark-fg-tertiary:rgb(255 255 255 / 0.4);--color-app-light-bg-primary:rgb(255 255 255 / 1);--color-app-light-bg-primary-hover:rgb(240 240 240 / 1);--color-app-light-bg-secondary:rgb(240 240 240 / 1);--color-app-light-bg-tertiary:rgb(221 221 221 / 1);--color-app-light-bg-quaternary:rgb(183 183 183 / 1);--color-app-dark-bg-primary:rgb(22 23 24 / 1);--color-app-dark-bg-primary-hover:rgb(27 28 29 / 1);--color-app-dark-bg-secondary:rgb(35 37 37 / 1);--color-app-dark-bg-tertiary:rgb(54 55 55 / 1);--color-app-dark-bg-quaternary:rgb(90 91 91 / 1);" class="pencraft pc-display-contents pc-reset appTheme-xUAu9p"><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer profile-themed-background"><div class="container"><div class="footer-terms"><span>© 2026 Charity Majors</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&utm_medium=web&utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="HnqQfs3nO8pxxyyQ577RVQ">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
|
||
</div>
|
||
|
||
|
||
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
|
||
<script nonce="HnqQfs3nO8pxxyyQ577RVQ">
|
||
window.Sentry && window.Sentry.onLoad(function() {
|
||
window.Sentry.init({
|
||
environment: window._preloads.sentry_environment,
|
||
dsn: window._preloads.sentry_dsn,
|
||
})
|
||
})
|
||
</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script nonce="HnqQfs3nO8pxxyyQ577RVQ">window._preloads = JSON.parse("{\"cspNonce\":\"HnqQfs3nO8pxxyyQ577RVQ\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://charity.wtf\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":32306597,\"byline_images_enabled\":true,\"bylines_enabled\":true,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Charity Majors\",\"cover_photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/6e483de5-cf92-41be-beb1-b5af853e75f9_555x555.png\",\"created_at\":\"2024-08-26T05:48:54.313Z\",\"custom_domain_optional\":false,\"custom_domain\":\"charity.wtf\",\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":null,\"default_show_guest_bios\":true,\"email_banner_url\":null,\"email_from_name\":\"Charity Majors\",\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":null,\"free_subscription_benefits\":null,\"ga_pixel_id\":null,\"google_site_verification_token\":null,\"google_tag_manager_token\":null,\"hero_image\":null,\"hero_text\":\"observability, tech advice, honeycomb.io, etc\",\"hide_intro_subtitle\":null,\"hide_intro_title\":null,\"hide_podcast_feed_link\":false,\"homepage_type\":\"profile\",\"id\":2935724,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":null,\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!Tg4H!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png\",\"minimum_group_size\":2,\"moderation_enabled\":true,\"name\":\"charity.wtf\",\"paid_subscription_benefits\":null,\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"disabled\",\"paywall_free_trial_enabled\":false,\"podcast_art_url\":null,\"paid_podcast_episode_art_url\":null,\"podcast_byline\":null,\"podcast_description\":null,\"podcast_enabled\":false,\"podcast_feed_url\":null,\"podcast_title\":null,\"post_preview_limit\":null,\"primary_user_id\":32306597,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"charitydotwtf\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#FF6719\",\"theme_var_color_links\":false,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":null,\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":null,\"stripe_user_id\":null,\"stripe_country\":null,\"stripe_publishable_key\":null,\"stripe_platform_account\":null,\"automatic_tax_enabled\":null,\"author_name\":\"Charity Majors\",\"author_handle\":\"charitydotwtf\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!EAp-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg\",\"author_bio\":\"cofounder and CTO of honeycomb.io; pioneered modern observability. co-author of O'Reilly books \\\"Database Reliability Engineering\\\" and \\\"Observability Engineering\\\", now wrapping up the 2nd ed. loves free software, free speech, and peaty single malts.\",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":null,\"web_bg_color\":\"#ffffff\",\"cover_bg_color\":null,\"publication_id\":2935724,\"color_links\":null,\"font_preset_heading\":null,\"font_preset_body\":null,\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"newspaper\",\"home_posts\":\"list\",\"home_show_top_posts\":false,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":null,\"custom_footer\":null,\"social_media_links\":null,\"font_options\":null,\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":{\"first_thread_email_sent_at\":null,\"activated_at\":null,\"reader_thread_notifications_enabled\":false,\"create_thread_minimum_role\":\"contributor\",\"photo_replies_enabled\":true,\"boost_free_subscriber_chat_preview_enabled\":false,\"push_suppression_enabled\":false},\"default_group_coupon_percent_off\":null,\"default_group_coupon_include_founding\":null,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2015-12-27T01:57:32.000Z\",\"has_podcast\":false,\"has_free_podcast\":false,\"has_subscriber_only_podcast\":false,\"has_community_content\":true,\"rankingDetail\":\"Launched 2 years ago\",\"rankingDetailFreeIncluded\":\"Thousands of subscribers\",\"rankingDetailOrderOfMagnitude\":0,\"rankingDetailFreeIncludedOrderOfMagnitude\":1000,\"rankingDetailFreeSubscriberCount\":\"Over 6,000 subscribers\",\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u062A\u0645 \u0627\u0644\u0625\u0637\u0644\u0627\u0642 2 years ago\"},\"ca\":{\"rankingDetail\":\"S\u2019ha llan\u00E7at fa 2 anys\"},\"da\":{\"rankingDetail\":\"Lancering 2 \u00E5r\"},\"de\":{\"rankingDetail\":\"vor 2 Jahren gelauncht\"},\"es\":{\"rankingDetail\":\"Lanzado hace 2 a\u00F1os\"},\"fr\":{\"rankingDetail\":\"Lanc\u00E9 il y a 2 ann\u00E9es\"},\"id\":{\"rankingDetail\":\"Diluncurkan 2 tahun yang lalu\"},\"ja\":{\"rankingDetail\":\"\u958B\u59CB\u65E5 2\u5E74\u524D\"},\"ko\":{\"rankingDetail\":\"2\uB144 \uC804\uC5D0 \uCD9C\uC2DC\"},\"nb\":{\"rankingDetail\":\"Lansert 2 \u00E5r\"},\"nl\":{\"rankingDetail\":\"Gelanceerd 2 jaar geleden\"},\"pl\":{\"rankingDetail\":\"Uruchomiono 2 lat temu\"},\"pt\":{\"rankingDetail\":\"Lan\u00E7ado 2 anos\"},\"pt-br\":{\"rankingDetail\":\"Lan\u00E7ado 2 anos\"},\"en-gb\":{\"rankingDetail\":\"Launched 2 years ago\"},\"it\":{\"rankingDetail\":\"Lanciato 2 anni\"},\"tr\":{\"rankingDetail\":\"2 y\u0131l ba\u015Flat\u0131ld\u0131\"},\"sv\":{\"rankingDetail\":\"Lanserad 2 \u00E5r sedan\"},\"fi\":{\"rankingDetail\":\"Launched 2 vuotta\"},\"is\":{\"rankingDetail\":\"Launched 2 \u00E1r\"},\"zh-hans\":{\"rankingDetail\":\"\u4E8E2 \u5E74\u524D\u4E0A\u7EBF\"},\"zh-hant\":{\"rankingDetail\":\"\u65BC 2 \u5E74\u524D \u63A8\u51FA\"},\"en\":{\"rankingDetail\":\"Launched 2 years ago\"}},\"freeSubscriberCount\":\"6,000\",\"freeSubscriberCountOrderOfMagnitude\":\"6.3K+\",\"author_bestseller_tier\":0,\"author_badge\":{\"type\":\"subscriber\",\"tier\":5,\"accent_colors\":null},\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":false,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":null,\"primary_profile_name\":\"Charity Majors\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!EAp-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg\",\"no_follow\":false,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"free\",\"sections\":[],\"podcastTabInfo\":null,\"multipub_migration\":null,\"navigationBarItems\":[],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Charity Majors\",\"handle\":\"charitydotwtf\",\"role\":\"admin\",\"owner\":true,\"user_id\":32306597,\"photo_url\":\"https://substackcdn.com/image/fetch/$s_!EAp-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg\",\"bio\":\"cofounder and CTO of honeycomb.io; pioneered modern observability. co-author of O'Reilly books \\\"Database Reliability Engineering\\\" and \\\"Observability Engineering\\\", now wrapping up the 2nd ed. loves free software, free speech, and peaty single malts.\",\"status\":{\"bestsellerTier\":null,\"subscriberTier\":5,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"subscriber\",\"tier\":5,\"accent_colors\":null},\"subscriber\":null}}],\"threads_v2_enabled\":true,\"viralGiftsConfig\":null,\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://charity.wtf\",\"hostname\":\"charity.wtf\",\"is_on_substack\":false,\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"pageThemes\":{\"podcast\":null},\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#ff6719\",\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":232,\"g\":83,\"b\":0,\"a\":1},\"primary_elevated\":{\"r\":232,\"g\":83,\"b\":0,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":213,\"g\":66,\"b\":0,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":206.90096477355226,\"g\":206.90096477355175,\"b\":206.9009647735519,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#FF6719\",\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":86,\"b\":0,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"secondary\":{\"r\":134,\"g\":135,\"b\":135,\"a\":1},\"tertiary\":{\"r\":146,\"g\":146,\"b\":146,\"a\":1},\"accent\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"secondary\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"secondary_elevated\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"tertiary\":{\"r\":221,\"g\":221,\"b\":221,\"a\":1},\"quaternary\":{\"r\":183,\"g\":183,\"b\":183,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}},\"wordmark_bg\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"is_dark\":false}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[180,92,88],\"population\":2},\"DarkVibrant\":{\"rgb\":[16,117,82],\"population\":12},\"LightVibrant\":{\"rgb\":[220,177,157],\"population\":9},\"Muted\":{\"rgb\":[158,133,126],\"population\":555},\"DarkMuted\":{\"rgb\":[58,63,68],\"population\":331},\"LightMuted\":{\"rgb\":[210,190,169],\"population\":99}}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":false,\"customDomain\":\"charity.wtf\"},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":false,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":false,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":null,\"twitterPixelId\":null,\"parselyPixelId\":null,\"googleTagManagerToken\":null,\"googleAnalytics4Token\":null,\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":false,\"credit_token_enabled\":false,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":false,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":false,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":null,\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"\",\"notes_feed_enabled\":false,\"paywall_unlock_tokens\":false,\"post_preview_crop_gravity\":\"auto\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":false,\"reader_referrals_leaderboard_enabled\":false,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":false,\"local_currency_modal_seen\":false,\"local_payment_methods_modal_seen\":false,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":false,\"welcome_page_opt_out_text\":\"No thanks\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":false,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://charity.wtf/p/live-your-best-life-with-structured-events\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":181745827,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2022-08-15T06:32:29.000Z\",\"updated_at\":\"2025-12-16T00:58:55.877Z\",\"publication_id\":2935724,\"search_engine_description\":null,\"search_engine_title\":null,\"section_id\":null,\"should_send_free_preview\":false,\"show_guest_bios\":true,\"slug\":\"live-your-best-life-with-structured-events\",\"social_title\":null,\"subtitle\":\"\",\"teaser_post_eligible\":true,\"title\":\"Live Your Best Life With Structured Events\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"everyone\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":0,\"reactions\":{\"\u2764\":0},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"giving-good-feedback-consider-the-ratio\",\"next_post_slug\":\"rituals-for-engineering-teams\",\"cover_image\":null,\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":null,\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"If you\u2019re like most of us, you learned to debug as a baby engineer by way of printf(3).\",\"body_html\":\"<p>If you\u2019re like most of us, you learned to debug as a baby engineer by way of printf(3). By the time you were shipping code to production you had probably learned to instrument your code with a real metrics library. Maybe a tenth of us learned to use gdb and still step through functions on the regular. (I said <em>maybe</em>.)</p><p>Printing stuff to stdout is still the Swiss Army knife of tools. Always there when you reach for it, usually helps more than it does harm. (I said <em>usually.</em>)</p><p>And then! In case you\u2019ve been living under a rock, we recently went and blew up ye aulde monolythe, and in the process we \u2026 lost most of our single-process tools and techniques for debugging. Forget gdb; even <em>printf</em> doesn\u2019t work when you\u2019re hopping the network between functions.</p><p>If your tool set no longer works for you, friend, it\u2019s time to go all in. Maybe what you <em>wanted</em> was a faster horse, but it\u2019s time for a car, and the sooner you turn in your oats for gas cans and a spare tire, the better.</p><h2>Exercising Good Technical Judgment (When You Don\u2019t Have Any)</h2><p>If you\u2019re stuck trying to debug modern problems with pre-modern tooling, the first thing to do is stop digging the hole. Stop pushing good data after bad into formats and stores that aren\u2019t going to help you answer the right questions.</p><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!ApgF!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!ApgF!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 424w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 848w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!ApgF!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg\\\" width=\\\"236\\\" height=\\\"218\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/d3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":218,"width":236,"resizeWidth":null,"bytes":null,"alt":"0893d048d8361fe632b090b0429ad78b-rainbow-dash-rainbows-e1542789580565.jpg","title":null,"type":null,"href":null,"belowTheFold":false,"topImage":true,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"0893d048d8361fe632b090b0429ad78b-rainbow-dash-rainbows-e1542789580565.jpg\\\" title=\\\"0893d048d8361fe632b090b0429ad78b-rainbow-dash-rainbows-e1542789580565.jpg\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!ApgF!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 424w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 848w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!ApgF!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd3b50bc6-a95b-43b6-b8ba-400eee512e55_236x218.jpeg 1456w\\\" sizes=\\\"100vw\\\" fetchpriority=\\\"high\\\"></picture><div></div></div></a><p>In brief: if you aren\u2019t rolling out a solution based on <strong>arbitrarily wide, structured raw events that are unique and ordered and trace-aware </strong>and without any aggregation at write time, you are going to regret it. (If you aren\u2019t using OpenTelemetry, you are going to regret that, too.)</p><p>So just make the leap as soon as possible.</p><p>But let\u2019s rewind a bit. Let\u2019s start with observability.</p><h2>Observability: an introduction</h2><p>Observability is not a new word or concept, but the definition of observability as a specific technical term applied to software engineering is relatively new \u2014 about four years old. Before that, if you heard the term in softwareland it was only as a generic synonym for telemetry (\u201Cthere are three pillars of observability\u201D, in one annoying formulation) or team names (twitter, for example, has long had an \u201Cobservability team\u201D).</p><p>The term itself originates with <a href=\\\"https://en.wikipedia.org/wiki/Observability\\\">control theory:</a></p><p>\u201CIn control theory, observability is a measure of how well internal states of a system can be inferred from knowledge of its external outputs. The observability and controllability of a system are mathematical duals. The concept of observability was introduced by Hungarian-American engineer Rudolf E. K\u00E1lm\u00E1n for linear dynamic systems.[1][2]\u201D</p><p>But when applied to a software context, observability refers to how well you can understand and reason about your systems, just by interrogating them and inspecting their outputs with your tools. How well can you understand the <em>inside</em> of the system from the <em>outside</em>?</p><p>Achieving this relies your ability to ask brand new questions, questions you have never encountered and never anticipated \u2014 <em><strong>without shipping new code</strong>. </em>Shipping new code is cheating, because it means that you knew in advance what the problem was in order to instrument it.</p><h3>But what about monitoring?</h3><p>Monitoring has a long and robust history, but it has always been about watching your systems for failures you can define and expect. Monitoring is for known-unknowns, and setting thresholds and running checks against the system. <strong>Observability is about the</strong></p><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!RTFM!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!RTFM!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 424w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 848w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!RTFM!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg\\\" width=\\\"300\\\" height=\\\"169\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":169,"width":300,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" title=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!RTFM!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 424w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 848w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!RTFM!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2142ebad-d4a0-4f5a-88bb-84fed9ec192f_300x169.jpeg 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><p><strong> unknown-unknowns.</strong> Which requires a fundamentally different mindset and toolchain.</p><p>\u201CMonitoring is the action of observing and checking the behavior and outputs of a system and its components over time.\u201D \u2014 @grepory, in his talk \u201C<a href=\\\"https://speakerdeck.com/grepory/monitoring-is-dead\\\">Monitoring is Dead</a>\u201C.</p><p><strong>Monitoring is a third-person perspective on your software</strong>. It\u2019s not software explaining itself from the inside out, it\u2019s one piece of software checking up on another.</p><h3>Observability is for understanding complex, ephemeral, dynamic systems (not for debugging code)</h3><p>You don\u2019t use observability for stepping through functions; it\u2019s not a debugger. Observability is for swiftly identifying where in your system the error or problem is coming from, so you can debug it \u2014 by reproducing it, or seeing what it has in common with other erroring requests. You can think of observability as being like B.I. (business intelligence) tooling for software applications, in the way you engage in a lot of exploratory, open-ended data sifting to detect novel patterns and behaviors.</p><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!rm9f!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!rm9f!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 424w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 848w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 1272w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!rm9f!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png\\\" width=\\\"242\\\" height=\\\"220\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/f32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":220,"width":242,"resizeWidth":null,"bytes":null,"alt":"rainbow_dash___no_by_cptofthefriendship-d4erd69","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"rainbow_dash___no_by_cptofthefriendship-d4erd69\\\" title=\\\"rainbow_dash___no_by_cptofthefriendship-d4erd69\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!rm9f!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 424w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 848w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 1272w, https://substackcdn.com/image/fetch/$s_!rm9f!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff32ddb56-3e4e-4c29-8995-43fe7b719ada_300x273.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><p>Observability is often about swiftly isolating or tracking down the problem in your large, sprawling, far-flung, dynamic system. Because the hard part of distributed systems is rarely debugging the code, <strong>it\u2019s figuring out where the code you need to debug is</strong>.</p><p>The need for observability is often associated with microservices adoption, because they are prohibitively difficult to debug without service-level event oriented tooling \u2014 the kind you can get from Honeycomb and Lightstep.. and soon, I hope, many other vendors.</p><h2>Events are the building blocks of observability</h2><p>Ergh, another overloaded data term. What even is an \u201Cevent\u201D?</p><p>An observability \u201Cevent\u201D is a hop in the lifecycle of an end-to-end request. If a request executes code on three services separated by network hops before returning to the user, that request generated three observability \u201Cevents\u201D, each packed with context and details about that code running in that environment. These are also sometimes called \u201C<a href=\\\"https://stripe.com/blog/canonical-log-lines\\\">canonical log lines</a>\u201C. If you implemented tracing, each event may be a span in your trace.</p><p>If request ID #A897BEDC hits your edge, then your API service, then four more internal services, and twice connects to a db and runs a query, then request ID #A897BEDC generated 8 observability events \u2026 assuming you are in fact gathering observability data from the edge, the API, the internal services and the databases.</p><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!hhTU!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!hhTU!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 424w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 848w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 1272w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!hhTU!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png\\\" width=\\\"176\\\" height=\\\"167\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/f42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":167,"width":176,"resizeWidth":null,"bytes":null,"alt":"ponyfm-i7812-original","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"ponyfm-i7812-original\\\" title=\\\"ponyfm-i7812-original\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!hhTU!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 424w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 848w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 1272w, https://substackcdn.com/image/fetch/$s_!hhTU!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff42b54c6-6bc5-495a-81ab-53eae1500f75_300x284.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><p>This is an important caveat. <strong>We only gather observability events from services that we can and do introspect</strong>. If it\u2019s a black box to us, that hop cannot generate an observability event. So if request ID #A897BEDC also performed 20 cache lookups and called out to 8 external HTTP services and 2 managed databases, those 30 hops do <em>not</em> generate observability events (assuming you haven\u2019t instrumented the memcache service and have no instrumentation from those external services/dbs). Each request generates one event per request per service hop.**</p><p>(I also wrote about <a href=\\\"https://charity.wtf/2019/02/05/logs-vs-structured-events/\\\">logs vs structured events here</a>.)</p><h2><strong>Observability is a first-person narrative</strong>.</h2><p>We care primarily about self-reported status from the code as it executes the request path.</p><h2></h2><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!k_6J!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!k_6J!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 424w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 848w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 1272w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!k_6J!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png\\\" width=\\\"300\\\" height=\\\"186\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/d2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":186,"width":300,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" title=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!k_6J!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 424w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 848w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 1272w, https://substackcdn.com/image/fetch/$s_!k_6J!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd2b3d0b5-9611-41a2-bdfc-fddef27dad29_300x186.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><p>Instrumentation is your eyes and ears, explaining the software and its environment from the perspective of your code. Monitoring, on the other hand, is traditionally a third-person narrative \u2014 it\u2019s one piece of software checking up on another piece of software, with no internal knowledge of its hopes and dreams.</p><p>First-person narrative reports have the best potential for telling a reliable narrative. And more importantly, they <strong>map directly to user experience</strong> in a way that third-party monitoring does not and cannot.</p><h3>Events \u2026 must be structured.</h3><p>First, <strong>structure your goddamn data</strong>. You\u2019re a computer scientist, you\u2019ve got no business using text search to plow through terabytes of text.</p><h3>Events \u2026 are not just structured logs.</h3><p>Now, part of the reason people seem to think structured data is cost-prohibitive is that <strong>they\u2019re doing it wrong</strong>. They\u2019re still thinking about these like log lines. And while you <em>can</em> look at events like they\u2019re just really wide structured log lines that aren\u2019t flushed to disk, here\u2019s why you shouldn\u2019t: logs have decades of abhorrent associations and absolutely ghastly practices.</p><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!AFeN!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!AFeN!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 424w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 848w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 1272w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!AFeN!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png\\\" width=\\\"170\\\" height=\\\"171\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/e44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":171,"width":170,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" title=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!AFeN!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 424w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 848w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 1272w, https://substackcdn.com/image/fetch/$s_!AFeN!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe44bb2cb-2662-4f96-8f91-7626a3477907_298x300.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><p>Instead of bundling up and passing along one neat little pile of context, they\u2019re spewing log lines inside loops in their code and DDoS\u2019ing their own logging clusters.They\u2019re shitting out \u201Clog lines\u201D with hardly any dimensions so they\u2019re information-sparse and just straight up wasting the writes. And then to compensate for the sparseness and repetitiveness they just start logging the same exact nouns tens or hundreds of times over the course of the request, just so they can correlate or reconstruct some lousy request that they never should have blown up in the first place!</p><p>But they keep hearing they should be structuring their logs, so they pile structure on to their horrendous little strings, which pads every log line by a few bytes, so their bill goes up but they aren\u2019t getting any benefit! just paying more! What the hell, structuring is bull shit!</p><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!PZ-U!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!PZ-U!,w_424,c_limit,f_webp,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 424w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_848,c_limit,f_webp,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 848w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_1272,c_limit,f_webp,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 1272w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_1456,c_limit,f_webp,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!PZ-U!,w_1456,c_limit,f_auto,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif\\\" width=\\\"320\\\" height=\\\"320\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/e692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":196,"width":196,"resizeWidth":null,"bytes":null,"alt":"giphy","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"giphy\\\" title=\\\"giphy\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!PZ-U!,w_424,c_limit,f_auto,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 424w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_848,c_limit,f_auto,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 848w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_1272,c_limit,f_auto,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 1272w, https://substackcdn.com/image/fetch/$s_!PZ-U!,w_1456,c_limit,f_auto,q_auto:good,fl_lossy/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fe692e68e-0285-43e5-a89f-81cf227c3791_256x256.gif 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><p>Kittens. You need a fundamentally different approach to reap the considerable benefits of structuring your data.</p><p>But the difference between strings and structured data is ~basically the difference between grep and all of computer science. \uD83D\uDE1B</p><h3>Events \u2026 must be arbitrarily wide and dense with context.</h3><p>So the most effective way to structure your instrumentation, to get the absolute most bang for your buck, is to emit a single arbitrarily wide event per request per service hop. At Honeycomb, the maturely instrumented datasets that we see are often 200-500 dimensions wide. Here\u2019s an event that\u2019s just 20 dimensions wide:</p><pre><code>{ \\n\\n \\\"timestamp\\\":\\\"2018-11-20 19:11:56.910\\\",\\n \\\"az\\\":\\\"us-west-1\\\",\\n \\\"build_id\\\":\\\"3150\\\",\\n \\\"customer_id\\\":\\\"2310\\\",\\n \\\"durationMs\\\":167,\\n \\\"endpoint\\\":\\\"/api/v2/search\\\",\\n \\\"endpoint_shape\\\":\\\"/api/v2/search\\\",\\n \\\"fraud_dur\\\":131,\\n \\\"hostname\\\":\\\"app14\\\",\\n \\\"id\\\":\\\"f46691dfeda9ede4\\\",\\n \\\"mysql_dur\\\":\\\"\\\",\\n \\\"name\\\":\\\"/api/v2/search\\\",\\n \\\"parent_id\\\":\\\"\\\",\\n \\\"platform\\\":\\\"android\\\",\\n \\\"query\\\":\\\"\\\",\\n \\\"serviceName\\\":\\\"api\\\",\\n \\\"status_code\\\":200,\\n \\\"traceId\\\":\\\"f46691dfeda9ede4\\\",\\n \\\"user_id\\\":\\\"344310\\\",\\n \\\"error_rate\\\":0,\\n \\\"is_root\\\":\\\"true\\\"\\n}</code></pre><p>So a well-instrumented service should have hundreds of these dimensions, all bundled around the context of each request. And yet \u2014 and here\u2019s why events blow the pants off of metrics \u2014 even with hundreds of dimensions, it\u2019s still <em>just one write</em>. <strong>Adding more dimensions to your event is effectively free,</strong> it\u2019s still one write plus a few more bits.</p><p>Compare this to a metric-based systems, where you are often in the position of trying to predict whether a metric will be valuable enough to justify the extra write, because every single metric or tag you add contributes linearly to write amplification. Ever gotten billed tens of thousands of dollars for your custom metrics, or had to prune your list of useful custom metrics down to something affordable? (\u201CBUT THOSE ARE THE ONLY USEFUL ONES!\u201D, as every ops team wails)</p><h3></h3><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!o6VP!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!o6VP!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 424w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 848w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 1272w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!o6VP!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png\\\" width=\\\"300\\\" height=\\\"218\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/f1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":218,"width":300,"resizeWidth":null,"bytes":null,"alt":"","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" title=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!o6VP!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 424w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 848w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 1272w, https://substackcdn.com/image/fetch/$s_!o6VP!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff1b5e52e-34eb-4c8b-a48e-33ec68746239_300x218.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><h3>Events \u2026 must pass along the blob of context as the request executes</h3><p>As you can imagine, it can be a pain in the ass to keep passing this blob of information along the life of the request as it hits many services and databases. So at Honeycomb we do all the annoying parts for you with our integrations. You just install the go pkg or ruby gem or whatever, and under the hood we:</p><ol><li><p>initialize an empty debug event when the request enters that service</p></li><li><p>prepopulate the empty debug event with any and all interesting information that we already know or can guess. language type, version, environment, etc.</p></li><li><p>create a framework so you can just stuff any other details in there as easily as if you were printing it out to stdout</p></li><li><p>pass the event along and maintain its state until you are ready to error or exit</p></li><li><p>write the extremely wide event out to honeycomb</p></li></ol><p>Easy!</p><p>(Check out this killer talk from <a href=\\\"http://twitter.com/lyddonb\\\">@lyddonb</a> on \u2026 well everything you need to know about life, love and distributed systems is in here, but around the 12:00 mark he describes why this approach is mandatory. WATCH IT. <a href=\\\"https://www.youtube.com/watch?v=xy3w2hGijhE&feature=youtu.be\\\">https://www.youtube.com/watch?v=xy3w2hGijhE&feature=youtu.be</a>)</p><h3>Events \u2026 should collect context like sticky buns collect dust</h3><p>Other stuff you\u2019ll want to track in these structured blobs includes:</p><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!LfJi!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!LfJi!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 424w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 848w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 1272w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!LfJi!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png\\\" width=\\\"190\\\" height=\\\"221\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":221,"width":190,"resizeWidth":null,"bytes":null,"alt":"1225287_1370081029072_full","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"1225287_1370081029072_full\\\" title=\\\"1225287_1370081029072_full\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!LfJi!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 424w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 848w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 1272w, https://substackcdn.com/image/fetch/$s_!LfJi!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F92fac973-705f-4d32-95c1-ab8d7cf79dca_257x300.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><ol><li><p>Metadata like src, dst headers</p></li><li><p>The timing stats and contents of every network call (our beelines wrap all outgoing http calls and db queries automatically)</p></li><li><p>Every raw db query, normalized query family, execution time etc</p></li><li><p>Infra details like AZ, instance type, provider</p></li><li><p>Language/environment context like $lang version, build flags, $ENV variables</p></li><li><p>Any and all unique identifying bits you can get your grubby little paws on \u2014 request ID, shopping cart ID, user ID, request ID, transaction ID, any other ID \u2026 <strong>these are always the highest value data</strong> for debugging.</p></li><li><p>Any other useful application context. Service name, build id, ordering info, error rates, cache hit rate, counters, whatever.</p></li><li><p>Possibly the system resource state at this point in time. e.g. values from /proc/net/ipv4 stats</p></li></ol><p>Capture all of it. Anything that ever occurs to you <em>(\u201Cthis MIGHT be handy someday\u201D) \u2014 </em>don\u2019t even hesitate, just throw it on the pile. Collect it up in one rich fat structured blob.</p><h3>Events \u2026 must be unique, ordered, and traceable</h3><p>You need a unique request ID, and you need to propagate it through your stack in some way that preserves sequence. Once you have that, traces are just a beautiful visualization layer on top of your shiny event data.</p><h3>Events \u2026 must be stored raw.</h3><p>Because observability means you need to be able to ask any arbitrary new question of</p><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!F7aC!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!F7aC!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 424w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 848w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 1272w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!F7aC!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png\\\" width=\\\"181\\\" height=\\\"125\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":125,"width":181,"resizeWidth":null,"bytes":null,"alt":"Rainbow-Dash-is-not-amused-my-little-pony-friendship-is-magic-31088082-900-622","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"Rainbow-Dash-is-not-amused-my-little-pony-friendship-is-magic-31088082-900-622\\\" title=\\\"Rainbow-Dash-is-not-amused-my-little-pony-friendship-is-magic-31088082-900-622\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!F7aC!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 424w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 848w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 1272w, https://substackcdn.com/image/fetch/$s_!F7aC!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F62d09f72-8452-4f3d-9418-b26157d70c1b_300x207.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><p>your system without shipping new code, and aggregation is a one-way trip. Once you have aggregated your data and discarded the raw requests, you have destroyed your ability to ask new questions of that data forever. For Ever.</p><p><strong>Aggregation is a one-way trip</strong>. You can always, always derive your pretty metrics and dashboards and aggregates from structured events, and you can never go in reverse. Same for traces, same for logs. The structured event is the gold standard. Invest in it now, save your ass in the future.</p><p>It\u2019s only observability if you can ask new questions. And that means storing raw events.</p><h3>Events\u2026are richer than metrics</h3><p>There\u2019s always tradeoffs when it comes to data. Metrics choose to sacrifice context and connective tissue, and sometimes high cardinality support, which you need to correlate anomalies or track down outliers. They have a very small, efficient data format, but they sacrifice everything else by discarding all but the counter, gauge, etc.</p><p>A metric looks like this, by the way.</p><pre><code>{ metric: \\\"db.query.time\\\", value: 0.502, tags: Array(), type: set }</code></pre><p>That\u2019s it. It\u2019s just a name, a number and maybe some tags. You can\u2019t dig into the event and see what else was happening when that query was strangely slow. You can never get that information back after discarding it at write time.</p><p>But because they\u2019re so cheap, you can keep every metric for every request! Maybe. (Sometimes.) More often, what happens is they aggregate at write time. So you never actually get a value written out for an individual event, it smushes everything together that happens in the 1 second interval and calculates some aggregate values to write out. And that\u2019s all you can ever get back to.</p><p>With events, and their relative explosion of richness, we sacrifice our ability to store every single observability event about every request. At FB, every request generated hundreds of observability events as it made its way through the stack. Nobody, NOBODY is going to pay for an o11y stack that is hundreds of times as large as production. The solution to that problem is sampling.</p><h3>Events\u2026should be sampled.</h3><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!M0ok!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!M0ok!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 424w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 848w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 1272w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!M0ok!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png\\\" width=\\\"242\\\" height=\\\"220\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/87abc044-0ee7-413d-af84-819b99916fde_300x273.png","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":220,"width":242,"resizeWidth":null,"bytes":null,"alt":"rainbow_dash___no_by_cptofthefriendship-d4erd69","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"rainbow_dash___no_by_cptofthefriendship-d4erd69\\\" title=\\\"rainbow_dash___no_by_cptofthefriendship-d4erd69\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!M0ok!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 424w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 848w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 1272w, https://substackcdn.com/image/fetch/$s_!M0ok!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87abc044-0ee7-413d-af84-819b99916fde_300x273.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><p>But not dumb, blunt sampling at server side. Control it on the client side.</p><p>Then sample heavily for events that are known to be common and useless, but keep the events that have interesting signal. For example: health checks that return 200 OK usually represent a significant chunk of your traffic and are basically useless, while 500s are almost always interesting. So are all requests to /login or /payment endpoints, so keep all of them. For database traffic: SELECTs for health checks are useless, DELETEs and all other mutations are rare but you should keep all of them. Etc.</p><p>You don\u2019t need to treat your observability metadata with the same care as you treat your billing data. That\u2019s just dumb.</p><h2>\u2026 To be continued.</h2><p>I hope it\u2019s now blazingly obvious why observability requires \u2014 REQUIRES \u2014 that you have access to raw structured events with no pre-aggregation or write-time rollups. Metrics don\u2019t count. Just traces don\u2019t count. Unstructured logs sure as fuck don\u2019t count.</p><p>Structured, arbitrarily wide events, with dynamic sampling of the boring parts to control costs. There is no substitute.</p><p>For more about the technical requirements for observability, read <a href=\\\"https://www.honeycomb.io/blog/so-you-want-to-build-an-observability-tool/\\\">this</a>, <a href=\\\"https://thenewstack.io/observability-a-3-year-retrospective/\\\">this</a>, or <a href=\\\"https://thenewstack.io/observability-the-5-year-retrospective/\\\">this</a>.</p><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!emOQ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!emOQ!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 424w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 848w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!emOQ!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg\\\" width=\\\"195\\\" height=\\\"260\\\" data-attrs=\\\"{"src":"https://substack-post-media.s3.amazonaws.com/public/images/c2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg","srcNoWatermark":null,"fullscreen":null,"imageSize":null,"height":260,"width":195,"resizeWidth":null,"bytes":null,"alt":"IMG_4619","title":null,"type":null,"href":null,"belowTheFold":true,"topImage":false,"internalRedirect":null,"isProcessing":false,"align":null,"offset":false}\\\" class=\\\"sizing-normal\\\" alt=\\\"IMG_4619\\\" title=\\\"IMG_4619\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!emOQ!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 424w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 848w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 1272w, https://substackcdn.com/image/fetch/$s_!emOQ!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fc2796795-8afb-45fe-ab43-8eb4325a074f_225x299.jpeg 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a><p><em>**The deep fine print: it\u2019s one observability event per request per service hop \u2026 because we gather observability detail organized by request id. Databases may be different. For example, with MongoDB or MySQL, we can\u2019t instrument them to talk to honeycomb directly, so we gather information about its internal perspective by 1) tailing the slow query log (and turning it up to log all queries if perf allows), 2) streaming tcp over the wire and reconstructing transactions, 3) connecting to the mysql port as root every couple seconds from cron, then dumping all mysql stats and streaming them in to honeycomb as an event. SO. Database traffic is not organized around connection length or unique request id, it is organized around transaction id or query id. Therefore it generates one observability event per query or transaction. </em></p><p><em>In other words: if your request hit the edge, API, four internal services, two databases \u2026 but ran 1 query on one db and 10 queries on the second db \u2026 you would generate a total of *19 observability events* for this request.</em></p><p>For more on observability for databases and other black boxes, try<a href=\\\"https://www.honeycomb.io/blog/add-observability-to-databases/\\\"> this blog post</a>.</p>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"If you\u2019re like most of us, you learned to debug as a baby engineer by way of printf(3). By the time you were shipping code to production you had probably learned to instrument your code with a real metrics library. Maybe a tenth of us learned to use gdb and still step through functions on the regular. (I said\",\"wordcount\":2865,\"post_preview_limit\":null,\"language\":\"en\",\"postTags\":[],\"postCountryBlocks\":[],\"headlineTest\":null,\"publishedBylines\":[{\"id\":32306597,\"name\":\"Charity Majors\",\"handle\":\"charitydotwtf\",\"photo_url\":\"https://substackcdn.com/image/fetch/$s_!EAp-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg\",\"bio\":\"cofounder and CTO of honeycomb.io; pioneered modern observability. co-author of O'Reilly books \\\"Database Reliability Engineering\\\" and \\\"Observability Engineering\\\", now wrapping up the 2nd ed. loves free software, free speech, and peaty single malts.\",\"profile_set_up_at\":\"2022-11-24T22:50:45.155Z\",\"reader_installed_at\":\"2023-08-30T02:09:51.674Z\",\"publicationUsers\":[{\"id\":2985119,\"user_id\":32306597,\"publication_id\":2935724,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":2935724,\"name\":\"charity.wtf\",\"subdomain\":\"charitydotwtf\",\"custom_domain\":\"charity.wtf\",\"custom_domain_optional\":false,\"hero_text\":\"observability, tech advice, honeycomb.io, etc\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png\",\"author_id\":32306597,\"primary_user_id\":32306597,\"theme_var_background_pop\":\"#FF6719\",\"created_at\":\"2024-08-26T05:48:54.313Z\",\"email_from_name\":\"Charity Majors\",\"copyright\":\"Charity Majors\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"profile\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"is_guest\":false,\"bestseller_tier\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":5,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"subscriber\",\"tier\":5,\"accent_colors\":null},\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":0,\"comment_count\":0,\"child_comment_count\":0,\"is_geoblocked\":false,\"hasCashtag\":false},\"comments\":[],\"canonicalUrl\":\"https://charity.wtf/p/live-your-best-life-with-structured-events\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://charity.wtf/p/live-your-best-life-with-structured-events\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":null,\"background_pop\":\"#ff6719\",\"color_theme_bg_web\":null,\"cover_bg_color\":\"#FFFFFF\",\"cover_bg_color_secondary\":\"#f0f0f0\",\"background_pop_darken\":\"#ff5600\",\"print_on_pop\":\"#ffffff\",\"print_pop\":\"#ff6719\",\"color_theme_accent\":null,\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#ff6719\",\"background_contrast_1\":\"#f0f0f0\",\"background_contrast_2\":\"#dddddd\",\"background_contrast_3\":\"#b7b7b7\",\"background_contrast_4\":\"#929292\",\"background_contrast_5\":\"#515151\",\"color_theme_detail\":\"#e6e6e6\",\"background_contrast_pop\":\"rgba(255, 103, 25, 0.4)\",\"theme_bg_is_dark\":\"0\",\"background_pop_rgb\":\"255, 103, 25\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":null,\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"147.0.4074.113\",\"major\":\"147\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
|
||
<script nonce="HnqQfs3nO8pxxyyQ577RVQ">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"charitydotwtf\",\"publication_id\":2935724,\"has_plans\":false,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
|
||
|
||
|
||
|
||
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.7371871e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.571db0c6.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.a0731d37.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.d196682b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.2686bb5d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.ae8857e4.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.4614ce58.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.41564edd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.b2140fcd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.f3dc2b43.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.7aa24e85.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.1820773d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.765aca5a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.2f1bdde5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.4e726ad5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.f03f24de.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.f579d62b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.80c75c78.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.92b9fc9c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.258e49f2.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.d493457b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.ce61e623.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.015b0f71.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.f1207089.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.b91ab2ef.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.b93f6845.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89356.66161d1e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.211a7337.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47925.24a01d36.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.03bd4c41.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71505.3dc43e33.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.5d148f2c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.86d95e8c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.a5a682c3.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/86118.ff1ec843.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.7fe128ac.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.c78aabc1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.d6e9f8ec.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/29231.a7af5911.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.2b32bbb6.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.f1c17ded.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.1d0af2b9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.951c11cb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.58e078b0.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.dfde36c8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.94c6b3f1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.a327e9f4.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.8bb0d59c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.8f5c5991.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.cb326b80.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.680833b5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.df6a856d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.cf3bcc2c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.8a1bc5dd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.3f20fb36.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.bdbc4e21.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58483.3adddb8a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.0a97506f.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.09a9082f.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.7f0face7.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/92610.a4fc1daf.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.cb10b4b8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11244.b723f409.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99260.28b0ed6a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.7d5c5ba3.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/76768.9a2f2bae.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/41932.6c688f22.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12114.979fb38e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.9732c6f9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.4f5ae3be.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53871.2923c5c9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.588a06d1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99068.bd2e1fba.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.79d3f415.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.c7f428fb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.098bcbcf.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.2eb51db5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.72d1d844.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.bbf1c283.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.2e8ed950.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/32552.889dd584.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.c287f2d8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.6c265246.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.9b9761ae.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.b40bf629.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.07cd629b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.35b48b09.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.f37fb930.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.24075434.js" charset="utf-8"></script>
|
||
|
||
|
||
<script nonce="HnqQfs3nO8pxxyyQ577RVQ" nomodule>
|
||
(function() {
|
||
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
|
||
var warningDiv = document.createElement('div');
|
||
warningDiv.style.color = 'red';
|
||
warningDiv.style.padding = '10px';
|
||
warningDiv.style.margin = '10px 0';
|
||
warningDiv.style.border = '1px solid red';
|
||
warningDiv.style.backgroundColor = 'lightyellow';
|
||
warningDiv.innerText = message;
|
||
document.body.prepend(warningDiv);
|
||
})();
|
||
</script>
|
||
|
||
|
||
<!-- Datadog Analytics -->
|
||
<script nonce="HnqQfs3nO8pxxyyQ577RVQ">
|
||
(function(h,o,u,n,d) {
|
||
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
|
||
d=o.createElement(u);d.async=1;d.src=n
|
||
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
|
||
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
|
||
window.DD_RUM.onReady(function() {
|
||
window.DD_RUM.init({
|
||
clientToken: 'puba71073f072643721169b68f352438710',
|
||
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
|
||
site: 'datadoghq.com',
|
||
service: 'web',
|
||
env: window._preloads.dd_env,
|
||
version: '8b4c00716b2bc0f7ca1fded553014f7e37b5b07a',
|
||
sessionSampleRate: 1,
|
||
sessionReplaySampleRate: 10,
|
||
trackUserInteractions: window._preloads.dd_ti,
|
||
trackResources: true,
|
||
trackLongTasks: true,
|
||
defaultPrivacyLevel: 'mask-user-input',
|
||
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
|
||
});
|
||
})
|
||
</script>
|
||
<!-- End Datadog Analytics -->
|
||
|
||
<!-- Cloudflare Web Analytics -->
|
||
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
|
||
<!-- End Cloudflare Web Analytics -->
|
||
|
||
|
||
<!-- Fallback tracking pixels -->
|
||
|
||
|
||
|
||
|
||
<noscript>
|
||
<style>
|
||
#nojs-banner {
|
||
position: fixed;
|
||
bottom: 0;
|
||
left: 0;
|
||
padding: 16px 16px 16px 32px;
|
||
width: 100%;
|
||
box-sizing: border-box;
|
||
background: red;
|
||
color: white;
|
||
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
|
||
font-size: 13px;
|
||
line-height: 13px;
|
||
}
|
||
#nojs-banner a {
|
||
color: inherit;
|
||
text-decoration: underline;
|
||
}
|
||
</style>
|
||
|
||
<div id="nojs-banner">
|
||
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
|
||
</div>
|
||
</noscript>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="HnqQfs3nO8pxxyyQ577RVQ" data-cf-beacon='{"version":"2024.11.0","token":"68cfe66b5c4749e2ba64d4d9640c04c0","spa":2}' crossorigin="anonymous"></script>
|
||
</body>
|
||
</html>
|