4 lines
324 KiB
HTML
4 lines
324 KiB
HTML
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" href="/_next/static/media/345b029bff63acdd-s.p.14zxo1jbez.lc.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" href="/_next/static/media/8e6fa89aa22d24ec-s.p.0uvzar8hswo3p.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" href="/_next/static/media/ce62453a442c7f35-s.p.0333ktddfbsxy.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" href="/_next/static/media/e2334d715941921e-s.p.12skym0rqknxy.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" as="image" imageSrcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff45eaaf1f21497ecb581ef36b365c013d0be24d5-2561x2006.jpg&w=384&q=75 1x, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff45eaaf1f21497ecb581ef36b365c013d0be24d5-2561x2006.jpg&w=640&q=75 2x"/><link rel="stylesheet" href="/_next/static/chunks/0-a~7ljgpwyij.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/chunks/0k9no2apwmz~n.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/chunks/0nuksys438q2t.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/05gogik_6g8vk.js"/><script src="/_next/static/chunks/008dsykc_h8bq.js" async=""></script><script src="/_next/static/chunks/0i~~aj0ee08p-.js" async=""></script><script src="/_next/static/chunks/0qw_yv-45a3f4.js" async=""></script><script src="/_next/static/chunks/0._rkt7m1wq9e.js" async=""></script><script src="/_next/static/chunks/turbopack-16vnx6ub8-z5v.js" async=""></script><script src="/_next/static/chunks/14_mcc2-.c1ee.js" async=""></script><script src="/_next/static/chunks/019vi3l~x0zoc.js" async=""></script><script src="/_next/static/chunks/12~zlyzq33k~1.js" async=""></script><script src="/_next/static/chunks/025lo2k4b20~m.js" async=""></script><script src="/_next/static/chunks/0ku~nn_l8-fol.js" async=""></script><script src="/_next/static/chunks/039u5_5hlemld.js" async=""></script><script src="/_next/static/chunks/189hrhe.fk4af.js" async=""></script><script src="/_next/static/chunks/0oz-qxk30w~qw.js" async=""></script><script src="/_next/static/chunks/0vx_phl8cla2s.js" async=""></script><script src="/_next/static/chunks/0xywjwd04osrr.js" async=""></script><script src="/_next/static/chunks/0t1~yeqvn891r.js" async=""></script><script src="/_next/static/chunks/0ks4m351.jbv2.js" async=""></script><script src="/_next/static/chunks/01gy5f4286e6a.js" async=""></script><script src="/_next/static/chunks/0h3ejemwt0u_c.js" async=""></script><link rel="preload" href="https://cdn-cookieyes.com/client_data/497224d8995aa8565cbc3037/script.js" as="script"/><link rel="preload" href="/hny.min.js" as="script"/><link rel="preload" href="https://stats.honeycomb.io/js/pa-DJvf3AtvU5wDjKkEU-JVs.js" as="script"/><link rel="preload" href="https://www.googletagmanager.com/gtag/js?id=G-DJM5ZXPGB0" as="script"/><link rel="preload" href="https://www.googletagmanager.com/gtm.js?id=GTM-KH7NL6K" as="script"/><link rel="preload" href="https://cdn.jsdelivr.net/npm/hockeystack@latest/hockeystack.min.js" as="script"/><link rel="preconnect" href="https://cdn-cookieyes.com"/><link rel="dns-prefetch" href="https://cdn-cookieyes.com"/><meta name="next-size-adjust" content=""/><title>Incident Report: Exercises, Cleanups, and Evacuations</title><meta name="description" content="Every year, Honeycomb runs disaster recovery scenarios in multiple environments, including in production. Although each of our instances runs in a single region, on at least three Availability Zones (AZs), we have multiple plans for partial regional failures, and particularly, zonal failures."/><link rel="canonical" href="https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations"/><meta property="og:title" content="Incident Report: Exercises, Cleanups, and Evacuations"/><meta property="og:description" content="Every year, Honeycomb runs disaster recovery scenarios in multiple environments, including in production. Although each of our instances runs in a single region, on at least three Availability Zones (AZs), we have multiple plans for partial regional failures, and particularly, zonal failures."/><meta property="og:site_name" content="Honeycomb"/><meta property="og:locale" content="en_US"/><meta property="og:image" content="https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png?rect=0,77,3840,2006&w=1200&h=627&fit=crop"/><meta property="og:image:width" content="1200"/><meta property="og:image:height" content="627"/><meta property="og:type" content="article"/><meta property="article:published_time" content="2026-02-25T14:00:00.000Z"/><meta property="article:modified_time" content="2026-02-25T14:00:00.000Z"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:title" content="Incident Report: Exercises, Cleanups, and Evacuations"/><meta name="twitter:description" content="Every year, Honeycomb runs disaster recovery scenarios in multiple environments, including in production. Although each of our instances runs in a single region, on at least three Availability Zones (AZs), we have multiple plans for partial regional failures, and particularly, zonal failures."/><meta name="twitter:image" content="https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png?rect=0,77,3840,2006&w=1200&h=627&fit=crop"/><meta name="twitter:image:width" content="1200"/><meta name="twitter:image:height" content="627"/><link rel="icon" href="/favicon.ico?favicon.07o~09ebmt8eg.ico" sizes="48x48" type="image/x-icon"/><link rel="icon" href="/favicon-16x16.png" sizes="16x16" type="image/png"/><link rel="icon" href="/favicon-32x32.png" sizes="32x32" type="image/png"/><script>(self.__next_s=self.__next_s||[]).push([0,{"children":"window.ckySettings=window.ckySettings||{};window.ckySettings.nativeFunctions={fetch:window.fetch,xhr:window.XMLHttpRequest};","id":"cookieyes-preserve-native"}])</script><script>(self.__next_s=self.__next_s||[]).push(["https://cdn-cookieyes.com/client_data/497224d8995aa8565cbc3037/script.js",{"id":"cookieyes"}])</script><script referrerPolicy="no-referrer-when-downgrade" src="https://edge.wingify.net/tag/1272784.js"></script><script>(self.__next_s=self.__next_s||[]).push(["/hny.min.js",{}])</script><script>(self.__next_s=self.__next_s||[]).push([0,{"children":"\n window.Hny.initializeTracing({\n debug: false,\n apiKey: \"hcaik_01kc76rhfsq3kjf9tdmx6f7bmhhd0rzcdrpbtbgk31bpj4nezy2frmbqx8\",\n serviceName: \"www-next\",\n configDefaults: {\n ignoreNetworkEvents: true,\n propagateTraceHeaderCorsUrls: [\n /https:\\/\\/www\\.honeycomb\\.io/,\n /https:\\/\\/www-stage\\.honeycomb\\.io/,\n /https:\\/\\/honeycomb\\.vercel\\.app/,\n /https:\\/\\/honeycomb-stage\\.vercel\\.app/\n ]\n }\n });\n ","id":"hny-init"}])</script><script>(self.__next_s=self.__next_s||[]).push(["https://stats.honeycomb.io/js/pa-DJvf3AtvU5wDjKkEU-JVs.js",{"async":true,"id":"plausible-script"}])</script><script>(self.__next_s=self.__next_s||[]).push([0,{"children":"\n window.plausible = window.plausible || function() {\n (plausible.q = plausible.q || []).push(arguments)\n };\n plausible.init = plausible.init || function(i) {\n plausible.o = i || {}\n };\n plausible.init();\n ","id":"plausible-init"}])</script><script src="/_next/static/chunks/03~yq9q893hmn.js" noModule=""></script></head><body class="roboto_a526148e-module__zhi6Uq__className poppins_b750422f-module__FR8AEW__className roboto_459e8801-module__Bzopaa__className poppins_c92f7652-module__gE1l-G__className"><div hidden=""><!--$--><!--/$--></div><div class="jsx-62ee9791d3e4bddb bg-hc-cobalt font-roboto learn-more-header relative min-h-12 content-center py-2.5 text-center text-sm text-white"><span class="jsx-62ee9791d3e4bddb mr-2.5">Observability Engineering second edition out now! 27 net-new chapters written for today's observability challenges.</span><a target="_self" _type="link" class="font-roboto focus-visible:ring-ring inline-block cursor-pointer rounded-sm text-center font-bold uppercase transition-[color,background-color] [transition-duration:300ms,500ms] focus-visible:ring-1 focus-visible:outline-none disabled:pointer-events-none disabled:opacity-50 text-hc-cobalt hover:text-hc-pacific border-2 border-solid border-white bg-white px-2.5 py-1 text-[10px] leading-3.5 tracking-widest" href="/observability-engineering-oreilly-book"><span class="flex items-center justify-center gap-2.5">Get your copy</span></a></div><div class="jsx-62ee9791d3e4bddb bg-hc-cobalt fixed top-0 left-0 z-150 w-full text-white transition-all duration-300 ease-in-out pointer-events-none -translate-y-full opacity-0 pt-12"><form class="jsx-62ee9791d3e4bddb container mx-auto my-4 flex w-4/5 max-w-[1180px] items-center gap-4 border-b border-white px-2.5 pb-[5px]"><button type="button" aria-label="Search" class="jsx-62ee9791d3e4bddb"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke="white" class="h-4"><path d="M8.875 16.75C13.2242 16.75 16.75 13.2242 16.75 8.875C16.75 4.52576 13.2242 1 8.875 1C4.52576 1 1 4.52576 1 8.875C1 13.2242 4.52576 16.75 8.875 16.75Z" stroke="inherit" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path><path d="M14.4434 14.4438L18.9997 19.0002" stroke="inherit" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></svg></button><input type="text" placeholder="Search..." class="jsx-62ee9791d3e4bddb font-roboto flex-1 border-none bg-transparent text-base text-white outline-none placeholder:text-white/70" value=""/><button type="button" aria-label="Close search" class="jsx-62ee9791d3e4bddb text-white/70 transition-colors hover:text-white"><svg xmlns="http://www.w3.org/2000/svg" width="22" height="22" viewBox="0 0 22 22" fill="none" stroke="white" class="h-4"><path d="M0.999512 1L20.9992 20.9996" stroke="inherit" stroke-width="2" stroke-linecap="round"></path><path d="M20.9998 1L1.00011 20.9996" stroke="inherit" stroke-width="2" stroke-linecap="round"></path></svg></button></form></div><header id="main-navbar" style="opacity:var(--header-opacity);pointer-events:var(--header-pointer-events)" class="jsx-62ee9791d3e4bddb sticky top-0 z-50 w-full transition-transform duration-200 [&[scrolled]]:bg-white [&[scrolled]]:shadow-md"><div class="jsx-62ee9791d3e4bddb relative mx-auto flex max-w-[90%] items-center justify-between lg:h-full lg:max-w-[1180px] lg:px-6 xl:px-0"><a class="h-[105px] w-fit place-content-center" title="Honeycomb" href="/"><svg width="164" height="48" viewBox="0 0 164 48" fill="none" xmlns="http://www.w3.org/2000/svg" class="mb-5 lg:max-w-[135px] xl:max-w-none"><g clip-path="url(#clip0)"><path d="M151.695 26.0424C151.755 26.1018 151.8 26.1743 151.827 26.2541C151.861 26.333 151.879 26.4178 151.879 26.5036V27.1389C151.879 27.3115 151.818 27.4785 151.705 27.6094C151.652 27.667 151.588 27.7132 151.517 27.7456C151.445 27.7779 151.368 27.7956 151.29 27.7977H150.918C150.839 27.7986 150.761 27.7822 150.689 27.7496C150.618 27.7171 150.554 27.6692 150.503 27.6094C150.39 27.4785 150.328 27.3115 150.329 27.1389V26.5036C150.326 26.3305 150.388 26.1628 150.503 26.033C150.556 25.9753 150.62 25.9288 150.691 25.8965C150.762 25.8641 150.839 25.8465 150.918 25.8447H151.276C151.351 25.843 151.425 25.8591 151.493 25.8918C151.569 25.9285 151.638 25.9796 151.695 26.0424Z" fill="#25303E"></path><path d="M64.6293 28.8801H61.5331C61.0567 28.8741 60.5841 28.9668 60.1453 29.1523C59.7065 29.3378 59.311 29.612 58.9836 29.9577C58.6441 30.2861 58.3734 30.6786 58.1873 31.1125C58.0012 31.5463 57.9034 32.0128 57.8997 32.4848V40.0001H59.1438V33.0777C59.1533 32.2922 59.4655 31.5405 60.0156 30.9789C60.5644 30.4332 61.3054 30.124 62.0798 30.1177H63.9271C64.3091 30.1176 64.6872 30.1937 65.0392 30.3416C65.3913 30.4895 65.7102 30.7062 65.9771 30.9789C66.2546 31.2535 66.4746 31.5805 66.6242 31.9408C66.7739 32.3012 66.8503 32.6876 66.849 33.0777V40.0001H68.1497V32.4706C68.1547 31.9974 68.0609 31.5282 67.8743 31.0931C67.6877 30.658 67.4123 30.2665 67.0658 29.9436C66.7669 29.5936 66.3923 29.3159 65.9702 29.1317C65.5481 28.9474 65.0896 28.8614 64.6293 28.8801Z" fill="#25303E"></path><path d="M52.3573 28.6635H49.9161C48.8835 28.6805 47.8948 29.0835 47.1451 29.7929C46.4296 30.5294 46.0219 31.5104 46.0046 32.5364V36.3011C46.0122 37.3296 46.4135 38.3162 47.1262 39.0587C47.4865 39.4259 47.9181 39.7156 48.3947 39.9098C48.8713 40.104 49.3826 40.1988 49.8973 40.1882H52.3384C53.3813 40.1805 54.3822 39.7767 55.1377 39.0587C55.5023 38.7011 55.7911 38.2739 55.987 37.8026C56.1829 37.3313 56.2819 36.8255 56.2782 36.3152V32.5505C56.2646 31.5196 55.8565 30.533 55.1377 29.7929C54.3857 29.0813 53.3932 28.6782 52.3573 28.6635ZM54.9115 32.4282V36.5176C54.8979 37.1594 54.646 37.7732 54.2046 38.2399C53.7513 38.6749 53.1507 38.9235 52.5222 38.9364H49.8077C49.1756 38.9343 48.5697 38.6842 48.1206 38.2399C47.6725 37.7779 47.4209 37.1607 47.4184 36.5176V32.4282C47.4296 31.7882 47.6801 31.1755 48.1206 30.7105C48.5741 30.2724 49.1769 30.0219 49.8077 30.0093H52.5222C53.1507 30.0133 53.7521 30.2654 54.1952 30.7105C54.6405 31.1736 54.896 31.7864 54.9115 32.4282Z" fill="#25303E"></path><path d="M41.0003 28.8801H37.9135C37.4465 28.8641 36.9821 28.9559 36.5563 29.1483C36.1749 29.2725 35.8235 29.4744 35.5242 29.7412V24.8424H34.2754V40.0001H35.5808V33.0212C35.5904 32.2366 35.9008 31.4856 36.4479 30.9224C36.9983 30.3793 37.7383 30.0705 38.5121 30.0612H40.3594C40.7419 30.062 41.1204 30.1386 41.4731 30.2864C41.8258 30.4342 42.1456 30.6504 42.4141 30.9224C42.6897 31.1982 42.9081 31.5255 43.0569 31.8856C43.2056 32.2458 43.2819 32.6317 43.2812 33.0212V39.9624H44.5866V32.433C44.5919 31.9592 44.4978 31.4896 44.3103 31.0544C44.1228 30.6191 43.8461 30.2279 43.498 29.9059C43.1814 29.564 42.7939 29.295 42.3625 29.1179C41.9311 28.9407 41.4663 28.8595 41.0003 28.8801Z" fill="#25303E"></path><path d="M108.071 28.7059H105.63C104.596 28.7207 103.607 29.124 102.858 29.8353C102.142 30.5713 101.734 31.5526 101.718 32.5788V36.3435C101.735 37.3695 102.143 38.3505 102.858 39.0871C103.219 39.4542 103.65 39.7439 104.127 39.9381C104.604 40.1324 105.115 40.2271 105.63 40.2165H108.071C109.103 40.1985 110.091 39.7956 110.842 39.0871C111.557 38.3505 111.965 37.3695 111.982 36.3435V32.5788C111.966 31.5526 111.558 30.5713 110.842 29.8353C110.481 29.4681 110.05 29.1785 109.573 28.9842C109.097 28.79 108.585 28.6953 108.071 28.7059ZM110.625 32.4706V36.56C110.611 37.2003 110.359 37.8126 109.918 38.2776C109.701 38.5044 109.44 38.6839 109.151 38.8046C108.861 38.9253 108.549 38.9846 108.236 38.9788H105.516C104.885 38.9763 104.28 38.7242 103.834 38.2776C103.385 37.8182 103.133 37.202 103.132 36.56V32.4706C103.144 31.8293 103.394 31.2154 103.834 30.7482C104.287 30.3128 104.888 30.0626 105.516 30.0471H108.236C108.867 30.0506 109.471 30.3025 109.918 30.7482C110.359 31.2149 110.611 31.8288 110.625 32.4706Z" fill="#25303E"></path><path d="M128.433 28.9319H125.394C124.806 28.9346 124.227 29.0834 123.711 29.3648C123.248 29.616 122.844 29.9643 122.528 30.386C122.224 29.948 121.812 29.5962 121.331 29.3648C120.833 29.0903 120.275 28.9417 119.705 28.9319H116.774C115.834 28.9565 114.939 29.3425 114.277 30.0095C113.948 30.3406 113.691 30.7359 113.522 31.1708C113.353 31.6057 113.276 32.0707 113.296 32.5366V40.066H114.517V33.1813C114.511 32.794 114.585 32.4095 114.734 32.0519C114.87 31.7026 115.074 31.383 115.332 31.1107C115.61 30.8455 115.928 30.6263 116.275 30.4613C116.615 30.3097 116.986 30.2373 117.359 30.2495H119.098C119.752 30.2489 120.385 30.477 120.888 30.8942C121.38 31.2896 121.725 31.838 121.869 32.4519V40.1225H123.226V32.3201C123.398 31.7193 123.761 31.1904 124.26 30.8128C124.758 30.4352 125.366 30.2293 125.992 30.226H127.788C128.56 30.2331 129.3 30.5422 129.847 31.0872C130.397 31.626 130.71 32.3604 130.719 33.1295V40.0707H132.076V32.5413C132.079 32.073 131.989 31.6088 131.812 31.1752C131.635 30.7415 131.374 30.347 131.044 30.0142C130.33 29.3551 129.405 28.9714 128.433 28.9319Z" fill="#25303E"></path><path d="M142.444 29.7412C142.085 29.3913 141.659 29.1158 141.193 28.9309C140.726 28.7459 140.227 28.6551 139.725 28.6636H137.284C136.339 28.6603 135.428 29.0113 134.73 29.6471V24.7906H133.368V36.2354C133.387 37.2609 133.795 38.241 134.508 38.9789C134.869 39.3447 135.301 39.6335 135.778 39.8276C136.254 40.0218 136.765 40.1173 137.279 40.1083H139.72C140.234 40.113 140.744 40.0155 141.219 39.8216C141.695 39.6277 142.128 39.3412 142.491 38.9789C142.856 38.6213 143.146 38.1943 143.342 37.723C143.539 37.2518 143.639 36.7459 143.637 36.2354V32.4706C143.631 31.9738 143.538 31.4818 143.363 31.0165C143.119 30.5501 142.81 30.1207 142.444 29.7412ZM134.678 32.3765C134.691 31.7347 134.943 31.1209 135.385 30.6542C135.837 30.2171 136.438 29.9667 137.067 29.953H139.782C140.413 29.9565 141.017 30.2085 141.464 30.6542C141.914 31.1154 142.167 31.7327 142.171 32.3765V36.4659C142.158 37.1065 141.906 37.7191 141.464 38.1836C141.013 38.6226 140.411 38.8734 139.782 38.8848H137.067C136.754 38.884 136.445 38.8217 136.156 38.7014C135.867 38.5811 135.605 38.4051 135.385 38.1836C134.934 37.7246 134.681 37.1085 134.678 36.4659V32.3765Z" fill="#25303E"></path><path d="M99.4367 38.08C99.2387 38.3036 98.9985 38.4859 98.7298 38.6165C98.4414 38.7734 98.1154 38.8483 97.7873 38.833H95.2378C94.6185 38.7967 94.036 38.5277 93.6072 38.08C93.3933 37.8586 93.2256 37.5968 93.1139 37.31C93.0023 37.0232 92.9489 36.7171 92.9569 36.4095V32.3201C92.9502 31.7018 93.1832 31.1048 93.6072 30.6542C94.0432 30.2168 94.6216 29.9497 95.2378 29.9012H97.7873C98.3346 29.9003 98.8606 30.1131 99.2529 30.4942L99.3095 30.5459H100.851L100.794 30.3859C100.476 29.8387 100.03 29.3769 99.4933 29.0401C98.9457 28.7306 98.327 28.5684 97.6978 28.5695H95.4169C94.4297 28.6234 93.4992 29.0468 92.8108 29.7553C92.1288 30.4822 91.7584 31.4466 91.7787 32.4424V36.2071C91.7567 36.7018 91.837 37.1956 92.0145 37.6579C92.1921 38.1203 92.4632 38.5411 92.8108 38.8942C93.4915 39.6135 94.4266 40.039 95.4169 40.08H97.6978C98.3518 40.0648 98.9897 39.8747 99.5451 39.5295C100.113 39.2007 100.568 38.7087 100.851 38.1177L100.902 37.9577H99.4367V38.08Z" fill="#25303E"></path><path d="M85.8074 37.647L85.699 37.5952C85.3598 37.4943 85.0408 37.335 84.7564 37.1246C84.0922 36.4456 83.6066 35.613 83.3426 34.7011L81.6602 29.1105C81.6602 29.054 81.6084 29.0023 81.6084 28.9458C81.6084 28.8893 81.5518 28.8376 81.5518 28.7858V28.7058H80.303V28.8093C80.3563 29.1008 80.4287 29.3885 80.5198 29.6705L82.0419 34.847C82.5839 36.7293 83.7809 38.2352 85.19 38.7717L82.6593 43.7646H83.96L91.7264 28.7058H90.3692L85.8074 37.647Z" fill="#25303E"></path><path d="M151.747 29.3083H150.499V40.1319H151.747V29.3083Z" fill="#25303E"></path><path d="M163.675 30.9789C163.482 30.5057 163.184 30.0825 162.803 29.7413C162.438 29.3891 162.018 29.0983 161.559 28.8801C161.094 28.6869 160.593 28.5954 160.089 28.6119H157.648C156.615 28.6276 155.626 29.0308 154.877 29.7413C154.514 30.0976 154.227 30.5225 154.031 30.9912C153.835 31.4599 153.735 31.9629 153.736 32.4707V36.2354C153.754 37.2614 154.161 38.2423 154.877 38.9789C155.237 39.3461 155.669 39.6357 156.145 39.83C156.622 40.0242 157.133 40.1189 157.648 40.1083H160.089C161.122 40.0935 162.112 39.6902 162.86 38.9789C163.576 38.2423 163.983 37.2614 164.001 36.2354V32.4707C164.001 31.9559 163.89 31.4471 163.675 30.9789ZM162.643 32.3907V36.4801C162.63 37.1206 162.378 37.7332 161.936 38.1977C161.485 38.6368 160.884 38.8876 160.254 38.8989H157.539C157.227 38.8982 156.917 38.8359 156.628 38.7156C156.34 38.5953 156.078 38.4193 155.857 38.1977C155.407 37.7388 155.153 37.1227 155.15 36.4801V32.3766C155.164 31.7348 155.416 31.1209 155.857 30.6542C156.31 30.2185 156.911 29.9697 157.539 29.9577H160.254C160.885 29.96 161.489 30.2102 161.936 30.6542C162.378 31.1209 162.63 31.7348 162.643 32.3766V32.3907Z" fill="#25303E"></path><path d="M76.4152 28.7058H73.0457C72.2864 28.683 71.5473 28.9525 70.9816 29.4588C70.405 29.95 70.0363 30.6411 69.9495 31.3929V36.2352C69.9469 36.7902 70.0593 37.3397 70.2794 37.8493C70.4865 38.3364 70.7858 38.7789 71.1607 39.1529C71.5273 39.5197 71.9742 39.7968 72.4661 39.9623C72.8202 40.0653 73.183 40.1362 73.55 40.174H75.7743C76.4868 40.1707 77.1804 39.9453 77.7584 39.5293C77.9509 39.3867 78.1321 39.2294 78.3003 39.0588C78.4445 38.8926 78.5723 38.7129 78.682 38.5223C78.7929 38.335 78.8829 38.1361 78.9507 37.9293L79.0072 37.8211L77.8243 37.647L77.7725 37.7035C77.6414 37.9878 77.4556 38.2436 77.2258 38.4564C76.851 38.7865 76.3728 38.9762 75.8733 38.9929H73.3756C73.1731 38.9895 72.9725 38.9529 72.7818 38.8846C72.3123 38.7352 71.9071 38.4317 71.6319 38.0235C71.4688 37.8209 71.3572 37.582 71.3067 37.327C71.2502 37.2188 71.2502 37.054 71.1984 36.9505V34.9599H76.4105C77.1952 34.9796 77.9558 34.6886 78.5265 34.1505C79.116 33.6067 79.4666 32.8524 79.502 32.0517V31.6752C79.4773 30.8789 79.1385 30.1245 78.5595 29.5764C78.2803 29.2931 77.9461 29.0697 77.5774 28.92C77.2087 28.7703 76.8132 28.6974 76.4152 28.7058ZM77.6075 33.4399C77.3228 33.6385 76.9784 33.7332 76.632 33.7082H71.2031V31.407C71.2475 31.0909 71.3779 30.7931 71.5801 30.5458C71.743 30.3474 71.9472 30.1868 72.1786 30.0752C72.4169 29.9676 72.6758 29.913 72.9373 29.9152H76.632C76.8754 29.913 77.1159 29.9678 77.3342 30.0752C77.5553 30.1872 77.7573 30.3333 77.9327 30.5082C78.1383 30.814 78.269 31.1638 78.3144 31.5293C78.3607 31.8921 78.3236 32.2606 78.2061 32.607C78.1024 32.946 77.8927 33.2429 77.6075 33.454V33.4399Z" fill="#25303E"></path><path d="M148.298 38.6119C148.297 38.9133 148.206 39.2076 148.038 39.4577C147.87 39.7078 147.631 39.9025 147.352 40.0172C147.073 40.1319 146.766 40.1614 146.47 40.1021C146.174 40.0427 145.903 39.8971 145.689 39.6837C145.476 39.4703 145.331 39.1986 145.273 38.903C145.214 38.6073 145.245 38.301 145.361 38.0227C145.476 37.7443 145.672 37.5065 145.923 37.3392C146.174 37.1718 146.469 37.0825 146.771 37.0825C146.972 37.0825 147.171 37.1221 147.356 37.199C147.542 37.2759 147.71 37.3887 147.852 37.5308C147.994 37.6728 148.106 37.8415 148.183 38.027C148.259 38.2125 148.298 38.4113 148.298 38.6119Z" fill="#25303E"></path><path d="M26.0654 32.7012L30.4387 40.3482L26.0654 48H17.3706L13.002 40.3482L17.3706 32.7012H26.0654Z" fill="#FFB000"></path><path d="M26.0654 14.24L30.4387 21.8871L26.0654 29.5388H17.3706L13.002 21.8871L17.3706 14.24H26.0654Z" fill="#64BA00"></path><path d="M10.6741 24.7906L14.2369 31.1154L10.6741 37.4495H3.56276L0 31.1154L3.56276 24.7906H10.6741Z" fill="#F96E10"></path><path d="M44.9918 0L50.647 10.0141L44.9918 20.0424H33.6249L27.9697 10.0141L33.6249 0H44.9918Z" fill="#0298EC"></path></g><defs><clipPath id="clip0"><rect width="164" height="48" fill="white"></rect></clipPath></defs></svg></a><nav class="jsx-62ee9791d3e4bddb flex gap-8 max-lg:hidden lg:items-center lg:gap-0"><div class="jsx-62ee9791d3e4bddb group lg:after:bg-hc-cobalt relative lg:px-1 lg:after:absolute lg:after:bottom-5 lg:after:left-0 lg:after:h-[2px] lg:after:w-0 lg:after:transition-[width] lg:after:duration-200 lg:after:ease-in-out lg:after:content-[''] first:lg:pl-0 lg:focus-within:after:mx-1 lg:focus-within:after:w-[calc(100%_-_8px)] lg:first:focus-within:after:mx-0 lg:first:focus-within:after:w-[calc(100%_-_4px)] lg:hover:after:mx-2 lg:hover:after:w-[calc(100%_-_8px)] lg:first:hover:after:mx-0 lg:first:hover:after:w-[calc(100%_-_4px)] xl:px-3.5 xl:focus-within:after:mx-3.5 xl:focus-within:after:w-[calc(100%_-_28px)] xl:first:focus-within:after:w-[calc(100%_-_14px)] xl:hover:after:mx-3.5 xl:hover:after:w-[calc(100%_-_28px)] xl:first:hover:after:w-[calc(100%_-_14px)]"><button aria-expanded="false" aria-haspopup="true" class="jsx-62ee9791d3e4bddb font-roboto flex w-full items-center justify-between text-left max-lg:cursor-default max-lg:text-xl max-lg:font-medium lg:pointer-events-none lg:block lg:py-10">Observability Platform<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></button><div class="jsx-62ee9791d3e4bddb menu fixed left-0 z-50 hidden w-full bg-[linear-gradient(to_right,_#f6f6f6_50%,_white_50%)] lg:top-[var(--navBottom,104px)] lg:group-focus-within:block lg:group-hover:block"><div class="jsx-62ee9791d3e4bddb container flex gap-12 xl:max-w-[1180px]"><div class="jsx-62ee9791d3e4bddb max-w-[170px] pt-12 pb-16"><h4 class="jsx-62ee9791d3e4bddb font-roboto mb-2 text-2xl font-bold">Explore the platform</h4><p class="jsx-62ee9791d3e4bddb mb-6 text-sm tracking-wide"> <!-- -->Honeycomb was built for the AI era. Learn how to futureproof your software for what comes next.</p><a target="_self" _type="link" class="font-roboto focus-visible:ring-ring inline-block cursor-pointer text-center font-bold tracking-[1.4px] uppercase transition-[color,background-color] [transition-duration:300ms,500ms] focus-visible:ring-1 focus-visible:outline-none disabled:pointer-events-none disabled:opacity-50 border-hc-cobalt text-hc-cobalt border-2 border-solid bg-transparent h-8 content-center rounded-md px-3 py-2 text-xs hover:border-hc-cobalt hover:bg-hc-cobalt hover:text-white" href="/platform"><span class="flex items-center justify-center gap-2.5">See overview</span></a></div><ul class="jsx-62ee9791d3e4bddb flex grow flex-wrap gap-x-4 bg-white pt-12 pb-16 pl-12 lg:gap-y-3 xl:gap-x-12"><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><a target="_self" _type="link" class="font-roboto border-hc-gray-400 group/arrow-link hover:text-hc-cobalt flex items-center border-b font-bold whitespace-nowrap lg:gap-2 lg:pb-3" href="/platform">Core Observability<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="transition-transform group-hover/arrow-link:translate-x-2 max-lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="3a7e355c0c21" _type="link" class="font-roboto" href="/platform/distributed-tracing">Distributed Tracing</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="57c3e96709fa" _type="link" class="font-roboto" href="/platform/log-analytics">Log Analytics</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="7c7fa92d4ab4" _type="link" class="font-roboto" href="/platform/metrics">Time Series Metrics</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="62cd2b120845" _type="link" class="font-roboto" href="/platform/frontend-observability">Frontend Observability</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="1c28ef24a5cd" _type="link" class="font-roboto" href="/platform/telemetry-pipeline">Telemetry Pipeline</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="a352c53a8036" _type="link" class="font-roboto" href="/platform/private-cloud">Private Cloud</a></li></ul></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><span class="jsx-62ee9791d3e4bddb font-roboto border-hc-gray-400 border-b font-bold whitespace-nowrap lg:pb-3">AI Agent Observability</span><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="6218cd7e67db" _type="link" class="font-roboto" href="/platform/agent-timeline">Agent Timeline</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="4f3399612de8" _type="link" class="font-roboto" href="/use-cases/ai-llm-observability">LLM Observability</a></li></ul></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><a target="_self" _type="link" class="font-roboto border-hc-gray-400 group/arrow-link hover:text-hc-cobalt flex items-center border-b font-bold whitespace-nowrap lg:gap-2 lg:pb-3" href="/platform/intelligence">Agentic Intelligence<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="transition-transform group-hover/arrow-link:translate-x-2 max-lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="bafda870cc04" _type="link" class="font-roboto" href="/platform/canvas">Canvas</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="e4c44c9abcc8" _type="link" class="font-roboto" href="/technologies/ai-agents">MCP</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="146b1517edde" _type="link" class="font-roboto" href="/blog/accelerate-opentelemetry-migrations-honeycomb-agent-skills">MCP Skills</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="89cfc92cfad2" _type="link" class="font-roboto" href="/blog/introducing-anomaly-detection-early-warning-system-service-health">Anomaly Detection</a></li></ul></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><span class="jsx-62ee9791d3e4bddb font-roboto border-hc-gray-400 border-b font-bold whitespace-nowrap lg:pb-3">Built-in Features</span><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="043ac481e220" _type="link" class="font-roboto" href="/platform/slos">SLOs</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="ad585cbbd043" _type="link" class="font-roboto" href="/platform/service-map">Service Map</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="bafe0c86bbaf" _type="link" class="font-roboto" href="/platform/bubbleup">BubbleUp</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="e8757ff0a28a" _type="link" class="font-roboto" href="/platform/opentelemetry">OpenTelemetry</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="d1f9cc290db7" _type="link" class="font-roboto" href="/platform/integrations">App Integrations</a></li></ul></li></ul><div class="jsx-62ee9791d3e4bddb"></div></div></div></div><div class="jsx-62ee9791d3e4bddb group lg:after:bg-hc-cobalt relative lg:px-1 lg:after:absolute lg:after:bottom-5 lg:after:left-0 lg:after:h-[2px] lg:after:w-0 lg:after:transition-[width] lg:after:duration-200 lg:after:ease-in-out lg:after:content-[''] first:lg:pl-0 lg:focus-within:after:mx-1 lg:focus-within:after:w-[calc(100%_-_8px)] lg:first:focus-within:after:mx-0 lg:first:focus-within:after:w-[calc(100%_-_4px)] lg:hover:after:mx-2 lg:hover:after:w-[calc(100%_-_8px)] lg:first:hover:after:mx-0 lg:first:hover:after:w-[calc(100%_-_4px)] xl:px-3.5 xl:focus-within:after:mx-3.5 xl:focus-within:after:w-[calc(100%_-_28px)] xl:first:focus-within:after:w-[calc(100%_-_14px)] xl:hover:after:mx-3.5 xl:hover:after:w-[calc(100%_-_28px)] xl:first:hover:after:w-[calc(100%_-_14px)]"><button aria-expanded="false" aria-haspopup="true" class="jsx-62ee9791d3e4bddb font-roboto flex w-full items-center justify-between text-left max-lg:cursor-default max-lg:text-xl max-lg:font-medium lg:pointer-events-none lg:block lg:py-10">Solutions<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></button><div class="jsx-62ee9791d3e4bddb menu fixed left-0 z-50 hidden w-full bg-[linear-gradient(to_right,_#f6f6f6_50%,_white_50%)] lg:top-[var(--navBottom,104px)] lg:group-focus-within:block lg:group-hover:block"><div class="jsx-62ee9791d3e4bddb container flex gap-12 xl:max-w-[1180px]"><div class="jsx-62ee9791d3e4bddb max-w-[170px] pt-12 pb-16"><h4 class="jsx-62ee9791d3e4bddb font-roboto mb-2 text-2xl font-bold">Why Honeycomb</h4><p class="jsx-62ee9791d3e4bddb mb-6 text-sm tracking-wide"> <!-- -->Discover why Honeycomb is the better choice for your engineers, your customers, and your bottom line.</p><a target="_self" _type="link" class="font-roboto focus-visible:ring-ring inline-block cursor-pointer text-center font-bold tracking-[1.4px] uppercase transition-[color,background-color] [transition-duration:300ms,500ms] focus-visible:ring-1 focus-visible:outline-none disabled:pointer-events-none disabled:opacity-50 border-hc-cobalt text-hc-cobalt border-2 border-solid bg-transparent h-8 content-center rounded-md px-3 py-2 text-xs hover:border-hc-cobalt hover:bg-hc-cobalt hover:text-white" href="/why-honeycomb"><span class="flex items-center justify-center gap-2.5">Learn More</span></a></div><ul class="jsx-62ee9791d3e4bddb flex grow flex-wrap gap-x-4 bg-white pt-12 pb-16 pl-12 lg:gap-y-3 xl:gap-x-12"><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><span class="jsx-62ee9791d3e4bddb font-roboto border-hc-gray-400 border-b font-bold whitespace-nowrap lg:pb-3">Technologies</span><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="973236213b56" _type="link" class="font-roboto" href="/platform/opentelemetry">OpenTelemetry</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="31d4054b0539" _type="link" class="font-roboto" href="/technologies/aws">Amazon Web Services</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="e4533f4323a3" _type="link" class="font-roboto" href="/technologies/microsoft-azure">Microsoft Azure</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="37dbd8aeb8c4" _type="link" class="font-roboto" href="/technologies/kubernetes">Kubernetes</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="6413768f27dc" _type="link" class="font-roboto" href="/technologies/google-cloud">Google Cloud</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="e3ba65b3e56a" _type="link" class="font-roboto" href="/technologies/ai-agents">AI Agents</a></li></ul></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><span class="jsx-62ee9791d3e4bddb font-roboto border-hc-gray-400 border-b font-bold whitespace-nowrap lg:pb-3">Use Cases</span><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="12dfc27e1b09" _type="link" class="font-roboto" href="/use-cases/ai-llm-observability">LLM Observability</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="ag3nt0bs3rv01" _type="link" class="font-roboto" href="/use-cases/agent-observability">Agent Observability</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="cdb5d85b653f" _type="link" class="font-roboto" href="/use-cases/incident-response">Incident Response</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="05d3787884f2" _type="link" class="font-roboto" href="/use-cases/predictable-costs">Predictable Costs</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="d56b2265ff47" _type="link" class="font-roboto" href="/use-cases/devops-releases">DevOps & Releases</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="687be93e73cd" _type="link" class="font-roboto" href="/platform/frontend-observability">Frontend Development</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="fd9b0d71772c" _type="link" class="font-roboto" href="/use-cases/meet-customer-experience-slas">Meet Customer SLAs</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="64e59d6ab8aa" _type="link" class="font-roboto" href="/use-cases/cloud-migrations">Cloud Migrations</a></li></ul></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><span class="jsx-62ee9791d3e4bddb font-roboto border-hc-gray-400 border-b font-bold whitespace-nowrap lg:pb-3">Industries</span><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="ece052132a81" _type="link" class="font-roboto" href="/industries/ai-startups">AI Startups</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="8e6dab1d6286" _type="link" class="font-roboto" href="/industries/financial-services">Financial Services</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="440f49eafb36" _type="link" class="font-roboto" href="/industries/retail">Retail & Ecommerce</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="9cb0aa2dcebe" _type="link" class="font-roboto" href="/industries/software-technology">Software & Technology</a></li></ul></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><a target="_self" _type="link" class="font-roboto border-hc-gray-400 group/arrow-link hover:text-hc-cobalt flex items-center border-b font-bold whitespace-nowrap lg:gap-2 lg:pb-3" href="/why-honeycomb">Why Honeycomb?<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="transition-transform group-hover/arrow-link:translate-x-2 max-lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="91d95a07e165" _type="link" class="font-roboto" href="/why-honeycomb/customers">Customer Stories</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="05a8b16c4187" _type="link" class="font-roboto" href="/why-honeycomb/comparisons">Comparisons</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="eda3f9bc2fd1" _type="link" class="font-roboto" href="/why-honeycomb/enterprise">For Enterprise</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="711d9271d964" _type="link" class="font-roboto" href="/why-honeycomb/services">Honeycomb Services</a></li></ul></li></ul><div class="jsx-62ee9791d3e4bddb"></div></div></div></div><div class="jsx-62ee9791d3e4bddb group lg:after:bg-hc-cobalt relative lg:px-1 lg:after:absolute lg:after:bottom-5 lg:after:left-0 lg:after:h-[2px] lg:after:w-0 lg:after:transition-[width] lg:after:duration-200 lg:after:ease-in-out lg:after:content-[''] first:lg:pl-0 lg:focus-within:after:mx-1 lg:focus-within:after:w-[calc(100%_-_8px)] lg:first:focus-within:after:mx-0 lg:first:focus-within:after:w-[calc(100%_-_4px)] lg:hover:after:mx-2 lg:hover:after:w-[calc(100%_-_8px)] lg:first:hover:after:mx-0 lg:first:hover:after:w-[calc(100%_-_4px)] xl:px-3.5 xl:focus-within:after:mx-3.5 xl:focus-within:after:w-[calc(100%_-_28px)] xl:first:focus-within:after:w-[calc(100%_-_14px)] xl:hover:after:mx-3.5 xl:hover:after:w-[calc(100%_-_28px)] xl:first:hover:after:w-[calc(100%_-_14px)]"><button aria-expanded="false" aria-haspopup="true" class="jsx-62ee9791d3e4bddb font-roboto flex w-full items-center justify-between text-left max-lg:cursor-default max-lg:text-xl max-lg:font-medium lg:pointer-events-none lg:block lg:py-10">Learn <svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></button><div class="jsx-62ee9791d3e4bddb menu fixed left-0 z-50 hidden w-full bg-[linear-gradient(to_right,_#f6f6f6_50%,_white_50%)] lg:top-[var(--navBottom,104px)] lg:group-focus-within:block lg:group-hover:block"><div class="jsx-62ee9791d3e4bddb container flex gap-12 xl:max-w-[1180px]"><div class="jsx-62ee9791d3e4bddb max-w-[170px] pt-12 pb-16"><h4 class="jsx-62ee9791d3e4bddb font-roboto mb-2 text-2xl font-bold">Observability Engineering</h4><p class="jsx-62ee9791d3e4bddb mb-6 text-sm tracking-wide"> <!-- -->Start your journey with the definitive guide to observability. Download our complimentary ebook.</p><a target="_self" _type="link" class="font-roboto focus-visible:ring-ring inline-block cursor-pointer text-center font-bold tracking-[1.4px] uppercase transition-[color,background-color] [transition-duration:300ms,500ms] focus-visible:ring-1 focus-visible:outline-none disabled:pointer-events-none disabled:opacity-50 border-hc-cobalt text-hc-cobalt border-2 border-solid bg-transparent h-8 content-center rounded-md px-3 py-2 text-xs hover:border-hc-cobalt hover:bg-hc-cobalt hover:text-white" href="/observability-engineering-oreilly-book"><span class="flex items-center justify-center gap-2.5">Get your copy</span></a></div><ul class="jsx-62ee9791d3e4bddb flex grow flex-wrap gap-x-4 bg-white pt-12 pb-16 pl-12 lg:gap-y-3 xl:gap-x-12"><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><span class="jsx-62ee9791d3e4bddb font-roboto border-hc-gray-400 border-b font-bold whitespace-nowrap lg:pb-3">Engineers</span><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_blank" rel="noopener noreferrer" _key="0579b3261d13" _type="link" class="font-roboto" href="https://docs.honeycomb.io/">Docs</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="ed11462fee29" _type="link" class="font-roboto" href="/observability-engineering-oreilly-book">Observability Engineering</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_blank" rel="noopener noreferrer" _key="acf97f14bc60" _type="link" class="font-roboto" href="https://docs.honeycomb.io/get-started">Quickstart</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_blank" rel="noopener noreferrer" _key="a54f159c78a0" _type="link" class="font-roboto" href="https://docs.honeycomb.io/send-data/">Sending data</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_blank" rel="noopener noreferrer" _key="053ed6ac8255" _type="link" class="font-roboto" href="https://play.honeycomb.io/sandbox/tours">Sandbox</a></li></ul></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><a target="_self" _type="link" class="font-roboto border-hc-gray-400 group/arrow-link hover:text-hc-cobalt flex items-center border-b font-bold whitespace-nowrap lg:gap-2 lg:pb-3" href="/resources">Resource Center<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="transition-transform group-hover/arrow-link:translate-x-2 max-lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="104a20760fd8" _type="link" class="font-roboto" href="/blog">Blog</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="6d821088a12e" _type="link" class="font-roboto" href="/resources/getting-started">Getting Started</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="e4d103599e59" _type="link" class="font-roboto" href="/resources/guides">Technical Guides</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="84b6f5e31734" _type="link" class="font-roboto" href="/resources/case-studies">Case Studies</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="a7b9915d2b3b" _type="link" class="font-roboto" href="/resources/webinars">Webinars</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="a3e557190042" _type="link" class="font-roboto" href="/resources/whitepapers">Whitepapers</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="f6862a0a3b48" _type="link" class="font-roboto" href="/resources/product-videos">Product Videos</a></li></ul></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><span class="jsx-62ee9791d3e4bddb font-roboto border-hc-gray-400 border-b font-bold whitespace-nowrap lg:pb-3">Community</span><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="8a2f7421464f" _type="link" class="font-roboto" href="/events">Events</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_self" _key="eb8b17f096e8" _type="link" class="font-roboto" href="/office-hours">Office Hours</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_blank" rel="noopener noreferrer" _key="bb0360e4b1ad" _type="link" class="font-roboto" href="https://join.slack.com/t/honeycombpollinators/shared_invite/zt-3zl4f8i68-1PhyiwIIqZ0mCXtJjJcSsg">Pollinators Slack</a></li></ul></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><a target="_blank" rel="noopener noreferrer" _type="link" class="font-roboto border-hc-gray-400 group/arrow-link hover:text-hc-cobalt flex items-center border-b font-bold whitespace-nowrap lg:gap-2 lg:pb-3" href="https://academy.honeycomb.io/">Honeycomb Academy<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="transition-transform group-hover/arrow-link:translate-x-2 max-lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a><ul class="jsx-62ee9791d3e4bddb space-y-3"><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_blank" rel="noopener noreferrer" _key="b410785737ee" _type="link" class="font-roboto" href="https://academy.honeycomb.io/app/catalog">Course Catalog</a></li><li class="jsx-62ee9791d3e4bddb hover:text-hc-cobalt"><a target="_blank" rel="noopener noreferrer" _key="eba7082cbcd2" _type="link" class="font-roboto" href="https://academy.honeycomb.io/app/learning_paths">Learning Paths</a></li></ul></li></ul><div class="jsx-62ee9791d3e4bddb"></div></div></div></div><div class="jsx-62ee9791d3e4bddb group lg:after:bg-hc-cobalt relative lg:px-1 lg:after:absolute lg:after:bottom-5 lg:after:left-0 lg:after:h-[2px] lg:after:w-0 lg:after:transition-[width] lg:after:duration-200 lg:after:ease-in-out lg:after:content-[''] first:lg:pl-0 lg:focus-within:after:mx-1 lg:focus-within:after:w-[calc(100%_-_8px)] lg:first:focus-within:after:mx-0 lg:first:focus-within:after:w-[calc(100%_-_4px)] lg:hover:after:mx-2 lg:hover:after:w-[calc(100%_-_8px)] lg:first:hover:after:mx-0 lg:first:hover:after:w-[calc(100%_-_4px)] xl:px-3.5 xl:focus-within:after:mx-3.5 xl:focus-within:after:w-[calc(100%_-_28px)] xl:first:focus-within:after:w-[calc(100%_-_14px)] xl:hover:after:mx-3.5 xl:hover:after:w-[calc(100%_-_28px)] xl:first:hover:after:w-[calc(100%_-_14px)]"><button aria-expanded="false" aria-haspopup="true" class="jsx-62ee9791d3e4bddb font-roboto flex w-full items-center justify-between text-left max-lg:cursor-default max-lg:text-xl max-lg:font-medium lg:pointer-events-none lg:block lg:py-10">Company<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></button><div class="jsx-62ee9791d3e4bddb menu fixed left-0 z-50 hidden w-full bg-[linear-gradient(to_right,_#f6f6f6_50%,_white_50%)] lg:top-[var(--navBottom,104px)] lg:group-focus-within:block lg:group-hover:block"><div class="jsx-62ee9791d3e4bddb container flex gap-12 xl:max-w-[1180px]"><div class="jsx-62ee9791d3e4bddb max-w-[170px] pt-12 pb-16"><h4 class="jsx-62ee9791d3e4bddb font-roboto mb-2 text-2xl font-bold">Our mission</h4><p class="jsx-62ee9791d3e4bddb mb-6 text-sm tracking-wide"> <!-- -->Bring observability to every software engineer.</p><a target="_self" _type="link" class="font-roboto focus-visible:ring-ring inline-block cursor-pointer text-center font-bold tracking-[1.4px] uppercase transition-[color,background-color] [transition-duration:300ms,500ms] focus-visible:ring-1 focus-visible:outline-none disabled:pointer-events-none disabled:opacity-50 border-hc-cobalt text-hc-cobalt border-2 border-solid bg-transparent h-8 content-center rounded-md px-3 py-2 text-xs hover:border-hc-cobalt hover:bg-hc-cobalt hover:text-white" href="/about"><span class="flex items-center justify-center gap-2.5">About Us</span></a></div><ul class="jsx-62ee9791d3e4bddb flex grow flex-wrap gap-x-4 bg-white pt-12 pb-16 pl-12 lg:gap-y-3 xl:gap-x-12"><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><a target="_self" _type="link" class="font-roboto border-hc-gray-400 group/arrow-link hover:text-hc-cobalt flex items-center border-b font-bold whitespace-nowrap lg:gap-2 lg:pb-3" href="/about">About Us<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="transition-transform group-hover/arrow-link:translate-x-2 max-lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a><p class="jsx-62ee9791d3e4bddb font-roboto max-w-[138px] text-xs leading-5 tracking-wide">Learn about our company, mission and values.</p></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><a target="_self" _type="link" class="font-roboto border-hc-gray-400 group/arrow-link hover:text-hc-cobalt flex items-center border-b font-bold whitespace-nowrap lg:gap-2 lg:pb-3" href="/careers">Careers<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="transition-transform group-hover/arrow-link:translate-x-2 max-lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a><p class="jsx-62ee9791d3e4bddb font-roboto max-w-[138px] text-xs leading-5 tracking-wide">Come for the impact, stay for the culture.
|
||
</p></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><a target="_self" _type="link" class="font-roboto border-hc-gray-400 group/arrow-link hover:text-hc-cobalt flex items-center border-b font-bold whitespace-nowrap lg:gap-2 lg:pb-3" href="/news">News<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="transition-transform group-hover/arrow-link:translate-x-2 max-lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a><p class="jsx-62ee9791d3e4bddb font-roboto max-w-[138px] text-xs leading-5 tracking-wide">See Honeycomb's latest press releases, media, and more</p></li><li class="jsx-62ee9791d3e4bddb lg:flex lg:flex-col lg:gap-3"><a target="_self" _type="link" class="font-roboto border-hc-gray-400 group/arrow-link hover:text-hc-cobalt flex items-center border-b font-bold whitespace-nowrap lg:gap-2 lg:pb-3" href="/partners">Partners<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="transition-transform group-hover/arrow-link:translate-x-2 max-lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a><p class="jsx-62ee9791d3e4bddb font-roboto max-w-[138px] text-xs leading-5 tracking-wide">Learn more about becoming a Honeycomb partner.</p></li></ul><div class="jsx-62ee9791d3e4bddb"></div></div></div></div><a target="_self" _key="b519e18d3109" _type="link" class="font-roboto lg:after:bg-hc-cobalt hover:text-hc-cobalt relative max-lg:flex max-lg:items-center max-lg:justify-between max-lg:text-xl max-lg:font-medium lg:px-1 lg:py-10 lg:after:absolute lg:after:bottom-5 lg:after:left-0 lg:after:h-[2px] lg:after:w-0 lg:after:transition-[width] lg:after:duration-200 lg:after:ease-in-out lg:after:content-[''] lg:hover:after:w-full xl:px-[14px]" href="/pricing">Pricing<svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="lg:hidden"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a></nav><div id="custom-nav-button-target" class="jsx-62ee9791d3e4bddb flex items-center gap-5"><button aria-label="Search" class="jsx-62ee9791d3e4bddb search-icon"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke="#25303E"><path d="M8.875 16.75C13.2242 16.75 16.75 13.2242 16.75 8.875C16.75 4.52576 13.2242 1 8.875 1C4.52576 1 1 4.52576 1 8.875C1 13.2242 4.52576 16.75 8.875 16.75Z" stroke="inherit" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path><path d="M14.4434 14.4438L18.9997 19.0002" stroke="inherit" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></svg></button><a target="_blank" class="hover:text-hc-cobalt font-roboto login-button max-lg:hidden" href="https://ui.honeycomb.io/login">Login</a><a target="_self" _type="link" class="font-roboto focus-visible:ring-ring inline-block cursor-pointer text-center font-bold tracking-[1.4px] uppercase transition-[color,background-color] [transition-duration:300ms,500ms] focus-visible:ring-1 focus-visible:outline-none disabled:pointer-events-none disabled:opacity-50 border-hc-cobalt bg-hc-cobalt hover:bg-hc-pacific hover:border-hc-pacific border-2 border-solid text-white h-8 content-center rounded-md px-3 py-2 text-xs nav-button max-lg:hidden qdemo get-a-demo-button" href="/get-a-demo"><span class="flex items-center justify-center gap-2.5">Get a demo</span></a><a target="_self" _type="link" class="font-roboto focus-visible:ring-ring inline-block cursor-pointer text-center font-bold tracking-[1.4px] uppercase transition-[color,background-color] [transition-duration:300ms,500ms] focus-visible:ring-1 focus-visible:outline-none disabled:pointer-events-none disabled:opacity-50 border-hc-cobalt text-hc-cobalt hover:text-hc-pacific hover:border-hc-pacific border-2 border-solid bg-transparent h-8 content-center rounded-md px-3 py-2 text-xs nav-button max-lg:hidden" href="https://ui.honeycomb.io/signup"><span class="flex items-center justify-center gap-2.5">Start for free</span></a><button aria-label="Menu" class="jsx-62ee9791d3e4bddb menu-icon lg:hidden"><svg xmlns="http://www.w3.org/2000/svg" width="32" height="22" viewBox="0 0 32 22" fill="none"><path d="M1 1H31" stroke="#25303E" stroke-width="2" stroke-linecap="round"></path><path d="M1 11H31" stroke="#25303E" stroke-width="2" stroke-linecap="round"></path><path d="M1 21H31" stroke="#25303E" stroke-width="2" stroke-linecap="round"></path></svg></button></div><div class="jsx-62ee9791d3e4bddb mt-6 hidden justify-center gap-1 md:justify-start"><p class="jsx-62ee9791d3e4bddb">Already a Honeycomb customer?</p><a target="_blank" class="text-hc-cobalt underline" href="https://ui.honeycomb.io/login">Login</a></div></div></header><main class="mx-auto w-full px-6 py-12 md:container"><script type="application/ld+json">{"@context":"https://schema.org","@graph":[{"@type":"Article","@id":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#article","isPartOf":{"@id":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations"},"author":{"name":"Fred Hebert","@id":"https://www.honeycomb.io/#/schema/person/fred-hebert"},"headline":"Incident Report: Exercises, Cleanups, and Evacuations","datePublished":"2026-02-25T14:00:00+00:00","dateModified":"2026-03-05T14:11:38+00:00","mainEntityOfPage":{"@id":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations"},"wordCount":4695,"publisher":{"@id":"https://www.honeycomb.io/#organization"},"inLanguage":"en-US","image":{"@id":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#primaryimage"},"thumbnailUrl":"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png","keywords":["Incident Response"],"articleSection":["Incident Response"]},{"@type":"WebPage","@id":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations","url":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations","name":"Incident Report: Exercises, Cleanups, and Evacuations | Honeycomb","isPartOf":{"@id":"https://www.honeycomb.io/#website"},"inLanguage":"en-US","primaryImageOfPage":{"@id":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#primaryimage"},"image":{"@id":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#primaryimage"},"thumbnailUrl":"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png","datePublished":"2026-02-25T14:00:00+00:00","dateModified":"2026-03-05T14:11:38+00:00","description":"On December 5th, 2025, we suffered a major outage in our EU region, with the last recovery steps for it extending until December 17th, 2025. For multiple hours, all of Honeycomb’s event ingestion endpoints were down. Most of the duration was spent in a degraded mode where only Activity Log data was impacted. A general timeline is available on our status page, but in this report, we’ll look at a broader analysis of what happened.","breadcrumb":{"@id":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#breadcrumb"},"potentialAction":[{"@type":"ReadAction","target":["https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#primaryimage","url":"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png","contentUrl":"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png","width":3840,"height":2160,"caption":"Incident Report: Exercises, Cleanups, and Evacuations"},{"@type":"BreadcrumbList","@id":"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://www.honeycomb.io/"},{"@type":"ListItem","position":2,"name":"Incident Report: Exercises, Cleanups, and Evacuations"}]},{"@type":"WebSite","@id":"https://www.honeycomb.io/#website","url":"https://www.honeycomb.io/","name":"Honeycomb","description":"Event Driven Debugging","publisher":{"@id":"https://www.honeycomb.io/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https://www.honeycomb.io/search?searchTerm={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https://www.honeycomb.io/#organization","name":"Honeycomb","url":"https://www.honeycomb.io/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https://www.honeycomb.io/#/schema/logo/image/","url":"https://www.honeycomb.io/api/assets/honeycomb-logo.svg","contentUrl":"https://www.honeycomb.io/api/assets/honeycomb-logo.svg","width":164,"height":48,"caption":"Honeycomb"},"image":{"@id":"https://www.honeycomb.io/#/schema/logo/image/"},"sameAs":["https://x.com/honeycombio"]},{"@type":"Person","@id":"https://www.honeycomb.io/#/schema/person/fred-hebert","name":"Fred Hebert","url":"https://www.honeycomb.io/author/fred-hebert","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https://www.honeycomb.io/#/schema/person/image/fred-hebert","url":"https://cdn.sanity.io/images/927dxq0h/production/7b06f417ac780bd2e4f02707bab2ec76cfd56665-180x200.png","contentUrl":"https://cdn.sanity.io/images/927dxq0h/production/7b06f417ac780bd2e4f02707bab2ec76cfd56665-180x200.png","caption":"Fred Hebert"}}]}</script><article style="contain:layout style"><header class="mx-auto flex max-w-[1116px] flex-col items-start gap-12 md:gap-13"><div class="flex w-full flex-col justify-center gap-9 md:gap-[36px]"><div class="flex flex-col gap-[32px] md:gap-10"><div class="flex w-full"><nav aria-label="breadcrumb"><ol class="text-hc-slate flex flex-wrap items-center gap-1.5 text-xs break-words sm:gap-2.5"><li class="inline-flex items-center gap-1.5"><a class="font-roboto font-medium transition-colors hover:underline" href="/blog">Blog</a></li><li role="presentation" aria-hidden="true" class="[&>svg]:h-3.5 [&>svg]:w-3.5"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-chevron-right"><path d="m9 18 6-6-6-6"></path></svg></li><li class="inline-flex items-center gap-1.5"><span role="link" aria-disabled="true" aria-current="page" class="text-foreground font-roboto font-medium">Incident Report: Exercises, Cleanups, and Evacuations</span></li></ol></nav></div><div class="flex flex-col gap-[14px]"><h1 class="w-full text-center text-4xl font-medium tracking-tighter lg:text-6xl">Incident Report: Exercises, Cleanups, and Evacuations</h1></div></div><div class="flex w-full justify-center text-center"><p class="max-w-[872px]">On December 5th, 2025, we suffered a major outage in our EU region, with the last recovery steps for it extending until December 17th, 2025. For multiple hours, all of Honeycomb’s event ingestion endpoints were down. Most of the duration was spent in a degraded mode where only Activity Log data was impacted. A general timeline is available on our status page, but in this report, we’ll look at a broader analysis of what happened.</p></div></div><div class="flex w-full flex-col gap-3"><div class="flex w-full flex-col items-center justify-between gap-5 md:flex-row"><div class="flex flex-row items-center px-5 md:px-0"><div class="flex items-center gap-[14px]"><img alt="Fred Hebert" loading="lazy" width="80" height="80" decoding="async" data-nimg="1" class="bg-hc-gray-50 size-10 rounded-full" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F7b06f417ac780bd2e4f02707bab2ec76cfd56665-180x200.png%3Frect%3D0%2C10%2C180%2C180%26w%3D80%26h%3D80&w=96&q=75 1x, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F7b06f417ac780bd2e4f02707bab2ec76cfd56665-180x200.png%3Frect%3D0%2C10%2C180%2C180%26w%3D80%26h%3D80&w=256&q=75 2x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F7b06f417ac780bd2e4f02707bab2ec76cfd56665-180x200.png%3Frect%3D0%2C10%2C180%2C180%26w%3D80%26h%3D80&w=256&q=75"/><p class="text-hc-slate text-base"><span>By: </span><a class="text-hc-cobalt underline hover:no-underline" href="/author/fred-hebert">Fred Hebert</a></p></div><span class="text-hc-slate font-roboto text-base font-medium"><span class="ml-1"> | </span><span>February 25, 2026</span></span></div><div class="bg-hc-gray-100 h-0.5 w-full md:hidden"></div><div class="flex flex-wrap gap-3 px-5 md:px-0"><a href="/blog/category/incident-response"><div class="font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-gray-200 text-hc-slate">Incident Response</div></a></div></div><div class="bg-hc-gray-100 hidden h-0.5 w-full md:block"></div></div></header><div class="m-auto w-full max-w-[1116px] justify-between lg:flex lg:gap-[80px] lg:py-10"><div class="h-fit lg:sticky lg:top-28 lg:w-[280px]"><div class="hidden overflow-hidden rounded-[22px] bg-gray-50 lg:block"><img alt="Investigating Mysterious Kafka Broker I/O When Using Confluent Tiered Storage" width="280" height="184" decoding="async" data-nimg="1" class="h-[184px] object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff45eaaf1f21497ecb581ef36b365c013d0be24d5-2561x2006.jpg&w=384&q=75 1x, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff45eaaf1f21497ecb581ef36b365c013d0be24d5-2561x2006.jpg&w=640&q=75 2x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff45eaaf1f21497ecb581ef36b365c013d0be24d5-2561x2006.jpg&w=640&q=75"/><div class="p-[30px] pt-5"><div class="mb-3 flex items-center gap-4"><div class="font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-gray-200 text-hc-slate">Blog</div><span class="h-3.5 border-r border-r-black/30"></span><span class="font-medium font-roboto text-hc-slate/70 text-xs"><span>June 7, 2024</span></span></div><div class="space-y-4"><h4 class="font-roboto text-sm font-medium md:text-base">Investigating Mysterious Kafka Broker I/O When Using Confluent Tiered Storage</h4><p>Earlier this year, we upgraded from Confluent Platform 7.0.10 to 7.6.0. While the upgrade went smoothly, there was one thing that was different from previous upgrades: due to changes in the metadata format for Confluent’s Tiered Storage feature, all of our tiered storage metadata files had to be converted to a newer format.</p><a target="_self" _type="link" class="text-hc-slate hover:text-hc-cobalt group/arrow-link items-center gap-2 font-roboto focus-visible:ring-ring cursor-pointer rounded-sm text-center font-bold tracking-[1.4px] uppercase transition-[color,background-color] [transition-duration:300ms,500ms] focus-visible:ring-1 focus-visible:outline-none disabled:pointer-events-none disabled:opacity-50 flex !p-0 px-4 py-[13px] text-[14px] leading-[14px] [&_span]:text-hc-cobalt [&_svg]:text-hc-cobalt" href="/blog/investigating-kafka-tiered-storage"><span class="flex items-center justify-center gap-2.5">Read More</span><svg width="11" height="12" viewBox="0 0 11 12" fill="none" xmlns="http://www.w3.org/2000/svg" class="transition-transform group-hover/arrow-link:translate-x-2"><path d="M5 1L10 6L5 11" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a></div></div></div></div><div class="min-w-0 flex-1" style="content-visibility:auto;contain-intrinsic-size:auto 1500px"><img alt="Incident Report: Exercises, Cleanups, and Evacuations" fetchPriority="low" loading="lazy" width="3840" height="2160" decoding="async" data-nimg="1" class="my-5 block h-auto w-full max-w-full rounded-4xl object-center object-contain" style="color:transparent" sizes="100vw" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png&w=3840&q=75"/><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Every year, Honeycomb runs disaster recovery scenarios in multiple environments, including in production. Although each of our instances runs in a single region, on at least three Availability Zones (AZs), we have multiple plans for partial regional failures, and particularly, zonal failures. One of these tests was run on December 5th, and after its successful completion came its cleanup steps.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Because tolerating zonal failures creates and shifts workloads in new zones that are inherently imbalanced once the failure is over, some work is required to gradually shift workloads back in proper balance. During one of these steps, while following our runbook, we ended up purposefully destroying Kafka brokers. In all previous runs in pre-production environments, this step went fine. In production, however, it caused multiple topic partitions to go fully leaderless, which caused major problems at multiple levels:</p><ul data-testid="article-list-bullet" class="text-hc-slate not-prose font-roboto mt-6 list-disc pl-6 text-sm leading-normal font-normal tracking-normal lg:text-base"><li class="mb-2">The topics we use to transmit all ingested events had multiple partitions that were damaged, some of which irreversibly so;</li><li class="mb-2">The topics used by some of our consumer groups to track where they were at were partially damaged;</li><li class="mb-2">Metadata topics used by Kafka and the Confluent addons we use (such as those used to manage tiered storage) were also partially damaged, with some partitions fully broken.</li></ul><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The first element in the list was our top priority: each Honeycomb team and environment is assigned multiple partitions, and while we can sustain some partitions becoming unavailable, if all partitions assigned to a team are unavailable, then we fail to ingest any data for them. But worse than that is the damage these partitions had received.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">To be brief, Kafka maintains an internal set of offsets that are available to consumers for them to reliably consume data. Retriever, our query engine, takes these offsets and uses them in multiple places internally to track consistency of which events it has processed. If all Kafka brokers for a topic partition are lost, the ensuing leader election needs to be “dirty,” which can partially or completely roll back the offsets. This breaks all Retriever ingestion in the foreseeable future, possibly requiring to delete the environment and start again from scratch.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The early response then was split in two: find the teams and environments whose partition assignment needed to be moved (and impact communicated), and see if we could salvage the impacted partitions without offset/data loss.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Roughly one hour into the incident, the impacted customers were found. The Kafka recovery was a bit trickier: we couldn’t do <em>any</em> random leader election and had to salvage as much data as possible from the offsets that may still exist.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Through the alerting noise, we noticed that our ingest service, Shepherd, had entered a crashloop spiral more than an hour earlier due to increased memory buffering for events that were to be written on the broken partitions.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Various options to repair ingest for apps on broken partitions were considered, such as adding new ones to all datasets, booting Retrievers in the wrong partitions to create room, and so on. In the end, the impacted partitions were marked read-only on the producer side, which fixed ingest’s crashlooping. Reassignments were then run to bring everything back to normal for customers.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Later in the night, a list of leaderless Retriever partitions (all read-only) with their offsets was established. Roughly a third of the partitions were impacted, but a single one had complete data loss, roughly 2.5% of our cluster.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">As the system was then relatively stable but far from solid, a plan for the night and checking back on Saturday was put in place. The Kafka autobalancer was turned off, and a manual reassignment plan to bring back some balance was put in place, left to run overnight.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Completing reassignments to fresh hosts, for tiered storage specifically, can sometimes take multiple hours of idle-looking time to scan S3 buckets for historical data before any transfer would show up at the network level. We found it concerning to see disk usage going up, but decided to wait.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Around midnight PT, one partition still showed no sign of recovery: the disk was filling too fast on the brokers for comfort, and so local retention was reduced on Kafka, aiming to use the S3 tiered storage for any longer retention need. A status update was posted, and the group disbanded for the night.</p><h2 id="weekend-response" class="text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24">Weekend response</h2><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Early on Saturday morning, responders noticed that the Kafka balancing jobs did not progress, and disk usage hadn’t improved. Estimates showed we were losing 5% of disk capacity per hour, with only 30% free. To avoid running out of disk on brokers and potentially suffering major data loss that propagates to Retriever, plans were put in place to shut down all ingest at the ALB layer if we reached 95% usage. This left us five hours to figure out a way to free space.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">At that point, the main reassignments were cancelled, and there was an attempt to move smaller partitions to free up space. Responders found out that two topics had broken topic partitions that could be relevant: <code class="font-mono">_confluent-tier-state</code> and <code class="font-mono">__consumer_offsets</code>. The theory was that since we were trying to move topics and their data to tiered storage, having these broken could prevent proper offloading and uploading of data.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The moment these were repaired, triggers about our SLO processing fired, complaining of ~18h of late data. Our SLO processing service had been unable to see new data had come in based on how it implemented its consumption, and repairing the metadata topics unblocked it. It started catching up.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">After multiple efforts, at 7 a.m. PT, with 5.07% of disk storage free, ingest was turned off. Disk usage stabilized, but did not empty—which would have been our expectation if we were uploading older data to S3 and not replacing it with new ingest.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Honeycomb engineers decided to bring up new Kafka brokers, with new Retrievers, and create brand new partitions to send ingest to.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">As this was being rolled out, an engineer came in and suggested we turn off tiered storage altogether to unstick the cleanup process and allow freeing disk space. This is something that had worked in 2021 during an old outage. We turned off tiered storage and disk space was instantly restored.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">We reopened ingest, and customer impact ended then. Later in the day, as part of cleanup, the team stopped work to manually fix Retriever topic partitions that were fully damaged and truncated them, leaving them in read-only mode. This created a gap where older data was unavailable for querying as Retriever struggled with the offset reset, but it wasn’t lost and we would remediate this problem in the following days.</p><h2 id="second-week-response" class="text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24">Second week response</h2><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Since Kafka was still in a rough but functional state, most of Monday’s response focused on two elements:</p><ol data-testid="article-list-number" class="text-hc-slate not-prose font-roboto mt-6 list-decimal pl-6 text-sm leading-normal font-normal tracking-normal lg:text-base"><li class="mb-2">Extending the retention of the Honeycomb MySQL readonly replica’s binlog, since Activity Log could no longer publish events to Kafka.</li><li class="mb-2">Salvaging the damaged Kafka topic partition for Retriever.</li></ol><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The broken partition was fully offline. Being read-only, no traffic was being lost, but the historical data was not available for querying, although queries succeeded overall, just with partial results.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Regardless, a series of small tweaks and fixes were put in place to allow the Retriever nodes to idle live and let old data be queried despite “broken” offsets.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">On Tuesday, work was done to try and bring back tiered storage; this first required deleting all the old S3 data, which took multiple hours. We did not manage to turn tiered storage back on.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">In parallel, work was done to unfreeze builds that hadn’t moved since Friday. Activity Log repairs got delayed due to more urgent remediation efforts on Generally Available Honeycomb features, though data retention would be available until Friday. We attempted to repair their partitions on Wednesday. After failing to do dirty leader elections in Kafka, we decided to delete the topics involved and recreate them.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">This is when things got <em>weird</em>. The delete operation timed out, and suddenly, none of our Kafka brokers could do anything administrative other than list topics. Describing, reassigning, or modifying them would just time out. The cluster was in a far worse state than expected, and responders on the call became uncomfortable.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">To move past the feeling, we decided to split approaches: one lane to try and salvage the cluster with non-risky operations and ZooKeeper surgery, and one lane to plan a full evacuation of the cluster to a new one.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Managers and directors were called in to help align the rest of the organization on these efforts. Wednesday afternoon was spent preparing the evacuation efforts (described below) with the support of directors and engineering from all teams that would interact with Kafka either for operations, or through their product area.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Thursday involved the elaboration of multiple evacuation plans, and a division of work while Kafka SMEs kept trying to see what could safely be done about the cluster. Later in the day, even changes such as rebooting the Kafka controller host to shift its responsibility to another broker were seen as high-reward but high-risk, and because evacuation efforts could benefit from prolonged steady state, we tabled these efforts.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">On Friday, we realized we wouldn’t be ready in time for Activity Log data to be salvaged in the MySQL binlog, which had a maximal storage time of seven days. Since the feature is still in beta and rushing to save more data through alternative replication means would drag people away from the Kafka evacuation, we chose to try a hail mary approach suggested by AWS support where we’d set up a new replica that was one week old from a snapshot, freeze it with manual replication, and hope that this would trigger AWS’ feature where stalled replication extends binlog retention to up to 30 days.</p><h2 id="evacuation" class="text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24">Evacuation</h2><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">An evacuation would essentially ask that we consider the current Kafka cluster as no longer worth salvaging. A new Kafka cluster would be needed, with brand new topics and partitions, all empty with reset offsets. This, in turn, implies modifying or ensuring all of our software components can properly shift from the old to the new cluster without data loss, which would require deep code changes. For example, Retriever needed to be edited so that it was allowed to reset its offset tracking, something that would require changes in some of its internals and a lot of consideration for complicated scenarios.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">This sort of effort would be expected to take weeks if not months in normal time, but the incoming holidays break and an already tired team made the deadline short. It was also unclear if the cluster, as it stood in its unmanageable state, would be up for long enough to complete the migration.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">We therefore planned for five options presented summarily here, depending on how much of the preconditions we’d meet:</p><ol data-testid="article-list-number" class="text-hc-slate not-prose font-roboto mt-6 list-decimal pl-6 text-sm leading-normal font-normal tracking-normal lg:text-base"><li class="mb-2">The existing Kafka cluster dies at the current state of the world. We have a full outage and must recover from scratch. At least four to eight hours of hard downtime expected, higher ongoing costs.</li><li class="mb-2">The existing Kafka cluster dies but we have managed to make Retriever able to handle offset resets; we can recover from scratch without leaving old partitions behind.</li><li class="mb-2">The existing Kafka cluster dies while we were ready to migrate to a new cluster. We can reset it, but ingest would be down for one to two hours, querying estimated down for up to three to eight hours.</li><li class="mb-2">We’re ready to migrate most things except some minor elements we haven’t had the time to prepare. Zero downtime for ingest, one hour downtime for primary querying and alerting, but may take anywhere between two to eight hours of work—and we were unsure what the impact would be on less critical services.</li><li class="mb-2">Full migration as we’re ready. Similar to the previous option, but more testing has happened, we pick some slightly safer implementation details across all applications, and reduce uncertainty.</li></ol><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Roughly midday on Friday, the Storage team managed to make the offset reset mechanism work for Retriever. This made the first scenario no longer be a concern.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Our platform team had made good progress on the feature flags to switch readers and writers, and also made it possible to boot a second Kafka cluster without interfering with the other ones nor any other environments.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">As we progressed through the work on Friday and firefought on some parallel events in the US, we thought we could ambitiously hit the 4th or 5th scenario on Monday afternoon, with a full dress rehearsal in one of our EU pre-prod clusters.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The dress rehearsal went well with people from more than half a dozen teams on the call. There was some friction around code changes and builds that needed ironing out, but the plan mostly worked fine and was completed within four hours.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Tuesday required some work to fix issues where domain names for a new Kafka clusters’ NLB wouldn’t fit AWS’ limits, but we managed to fix everything in time. The migration ran in two hours with no significant problem and up to one hour of delayed signal at most.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">We cleaned up the older clusters, and caught up with bits of Activity Log in the EU region. As it turns out, the replica trick didn’t work. To minimize lost data, we booted an instance from Tuesday December 9th, and replicated from there. This kept data loss on the Activity Log from December 5th at ~6 p.m. ET to December 9th at ~6 p.m. ET, and the rest was saved.</p><h2 id="analysis" class="text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24">Analysis</h2><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">This incident was draining,<strong> described as “the worst incident of my career”</strong> by many. However, a great part of working for Honeycomb is how caring the organization is toward our incident response team. Frankly, it's at a level many of us hadn’t seen at prior companies.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Some engineers mentioned wanting to keep response sustainable, while others felt that it should have required faster escalation.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Coordination proved challenging. Before this happened, multiple senior engineers mentioned feeling like they automatically grabbed the lead when they entered the room, which they mentioned being something they need to think more about.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Communication with the public was challenging. We aren’t necessarily used to running such long events, and decided to be explicit about potential losses and pending recovery of Activity Log. While the situation was internally dire, the Activity Log is actually a beta feature; customers let us know that we were giving too much information about minor features.</p><h3 class="font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24">Documentation fragmentation issues</h3><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Many of the elements in play for this outage were already known. Similar outages had been seen in recent years, with high level plans for workarounds, RFCs and migrations that would ease up on these conditions, and processes to run similar exercises safely. Ultimately, these documents’ existence was either unknown (outside of the people who were involved in writing and reviewing them), or inaccessible (blocked by ineffective search on some document systems). Interviews with members of staff revealed a frustration with our documentation situation, spread across multiple areas.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">We see a need to better consolidate our information as we keep growing.</p><h3 class="font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24">Bumpable roadmap effects</h3><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">When you are given multiple competing tasks of high importance, the one with a deadline further out or a higher level of uncertainty is easier to defer than the one with a more urgent deadline or well-understood scope. This is often represented as the <a class="not-prose text-hc-cobalt mt-6 font-normal underline hover:no-underline" href="https://en.wikipedia.org/wiki/Time_management#Eisenhower_method">Eisenhower matrix</a>:</p><img data-testid="article-image" alt="Eisenhower Matrix" fetchPriority="low" loading="lazy" width="1280" height="1280" decoding="async" data-nimg="1" class="not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full" style="color:transparent" sizes="100vw" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa17a14ed08ff4fcfcd69fe58befadecb128fe435-1280x1280.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa17a14ed08ff4fcfcd69fe58befadecb128fe435-1280x1280.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa17a14ed08ff4fcfcd69fe58befadecb128fe435-1280x1280.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa17a14ed08ff4fcfcd69fe58befadecb128fe435-1280x1280.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa17a14ed08ff4fcfcd69fe58befadecb128fe435-1280x1280.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa17a14ed08ff4fcfcd69fe58befadecb128fe435-1280x1280.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa17a14ed08ff4fcfcd69fe58befadecb128fe435-1280x1280.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa17a14ed08ff4fcfcd69fe58befadecb128fe435-1280x1280.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa17a14ed08ff4fcfcd69fe58befadecb128fe435-1280x1280.png&w=3840&q=75"/><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24"><a class="not-prose text-hc-cobalt mt-6 font-normal underline hover:no-underline" href="https://commons.wikimedia.org/w/index.php?curid=141867635">By Cmglee - Own work, CC BY-SA 4.0</a></p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">When caught between a clear urgent project that has well-understood customer benefits, it becomes difficult to say no to it in favor of an internally controlled project whose benefits are going to be less obvious or more contextual.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">This gets even muddier when you have to prioritize multiple projects from the same quadrant. It’s at these levels where we can see teams with agency in their own schedules adapt to pressures by deferring or prioritizing some projects without going back up to leadership to arbiter common issues of this type.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">We could think of the bumpable mechanism as an extra dimension over that matrix: how much control and flexibility there is across competing projects, which makes it possible to reframe or reorder their respective urgency and importance.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">In general, being flexible with deadlines is a good thing. It lets us reprioritize as new information becomes available and as we find we need to adjust our plans. However, it appears to be challenging for teams to avoid repeatedly deferring more flexible pieces of work that are locally controlled or have wider margins. This is a convenient and rational way to locally manage tradeoffs.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">We cannot counter all bumping—it plays a useful role to our ability to adapt and adjust—but may want to seek ways of knowing when bumping can become a reflex more than a strategic tool.</p><h3 class="font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24">Sociotechnical misalignment</h3><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">A few years ago, Honeycomb was small enough that critical knowledge only existed within single engineers’ minds. We have since then taken measures to ensure all components have owners and that critical knowledge has multiple people behind it. While this helped, such measures need frequent adjusting.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Something interesting that came up in the incident is that many key interventions came from people who no longer own the components they helped with. People were able to help because they were here back when teams’ ownership was less well-defined, despite having since moved to different roles. That these people were all able to offer meaningful support is a <em>strength</em> of this organization. It also shows, however, that there is some drift or gap between the socio- and the -technical part of the system.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">For example, in creating one team to manage one part of the system and assigning ownership of the other elements to another team, any existing coupling between these components does not vanish. So while both teams can grow a stronger expertise on their respective piece of software, the details get lost:</p><img data-testid="article-image" alt="Implementation details" fetchPriority="low" loading="lazy" width="745" height="480" decoding="async" data-nimg="1" class="not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full" style="color:transparent" sizes="100vw" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F67a85245528ba4ebea8dc539d9e23a8fe599f03e-745x480.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F67a85245528ba4ebea8dc539d9e23a8fe599f03e-745x480.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F67a85245528ba4ebea8dc539d9e23a8fe599f03e-745x480.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F67a85245528ba4ebea8dc539d9e23a8fe599f03e-745x480.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F67a85245528ba4ebea8dc539d9e23a8fe599f03e-745x480.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F67a85245528ba4ebea8dc539d9e23a8fe599f03e-745x480.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F67a85245528ba4ebea8dc539d9e23a8fe599f03e-745x480.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F67a85245528ba4ebea8dc539d9e23a8fe599f03e-745x480.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F67a85245528ba4ebea8dc539d9e23a8fe599f03e-745x480.png&w=3840&q=75"/><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The challenge is that engineers need and often want to move to different work over time, both for growth and motivational reasons. Deep systems knowledge also has value, sometimes in very specific and infrequent situations. Both forces compete with each other, and compromises around quick training or leaving docs and runbooks do not fully replace thorough understanding, even if that’s only what teams have time for. This can be supported by good engineering practices, fighting tech debt, and solid observability to make it easier to get your bearings faster, but sometimes the business will, either implicitly or explicitly, accept the risk.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The principle is that the socio- parts (teams) may be more siloed than -technical parts. Isolating one without untangling the other will lead to drift, and drift to surprises. There is no easy fix for it, and incidents such as this one are great opportunities to bring some of that deep knowledge back, in a way that inherently feels practical to everyone involved.</p><h2 id="conclusion" class="text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24">Conclusion</h2><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">In wanting to move past a simple explanation of “don’t destroy the Kafka cluster next time”, we have to look at a very large set of influences. This incident involved a long term response during a stressful situation, including planning, implementing, and executing a full Kafka evacuation in less than five business days. We could only accomplish this through a very rapid reorganization within Honeycomb.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">We hope that this document provides a sufficient explanation behind how events unfolded for one of our biggest outages in recent memory. Documenting our response and some of its challenges can be of use for other organizations who may encounter similar situations.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">An additional hope of ours is that by highlighting some of the key elements that we identified in our reviews and were in play within our broader organizational context, we can provide insights that can also help other organizations. We think the patterns uncovered do have the ability to carry meaning elsewhere.</p><h2 id="appendix-i:-technical-explanations" class="text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24">Appendix I: Technical Explanations</h2><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">This section aims to explain the mechanisms involved in a Kafka cluster. On their own, they are not considered explanations of <em>why</em> the outage happened, but contributors to <em>how</em> its events unfold.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">We assume some familiarity with the concepts described in <a class="not-prose text-hc-cobalt mt-6 font-normal underline hover:no-underline" href="https://docs.confluent.io/kafka/introduction.html">Confluent's Introduction to Kafka</a>.</p><h3 class="font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24">What are Kafka offsets?</h3><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">On disk, Kafka will store data such that each message has an offset (a relative order since the start of the topic), a position (to skip around files more easily), a timestamp (to know when the data was written), and then the messages’ contents. These are put in a “log” on disk, broken down by topic and partition. Consumers can then ask to stream the log’s data, being handed the above data (including the offset and message).</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">In practice, consumers will generally ask for an offset bigger than the last one they had seen in the stream previously. If the log gets truncated (offsets reset at 0) consumers will often just start processing again from either the start or the end of the stream.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The timestamps can in theory be used to help make decisions, but are not accurate enough on their own to ensure ordering. Because clocks on computers are not reliable enough at a small enough granularity, it is possible for multiple messages to share the same timestamp, but not the same offsets.</p><h3 class="font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24">Why do offsets matter to Retriever?</h3><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Generally, Kafka consumers know about the offsets, but they don’t necessarily store them long term, outside of knowing where you were at.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">For example, every time you migrate a Kafka cluster (something kind of rare), offsets get reset and lost. To make that work in complex workloads, there are migration systems that will create topics that map the new cluster’s offsets to the old cluster’s offsets (or vice versa) in order to let consumers switch from one to the other transparently.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Overall, the expectation is that you use offsets to track where you’re at in a topic, and if the offsets reset, you can choose to either start from 0 again, or from the newest offset.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Retriever is one of these systems where each bit of data we consume from Kafka gets stored again into its own column format, both on disk and on S3. When doing this, Retriever also stores the last offset seen, once globally on its own, but also within each segment metadata file it has about any data it stores.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">This is a safety mechanism that ensures Retriever never skips ahead or back in time when it shouldn’t, which could result in duplicated or missing events, which could in turn mess with investigations or alerting.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The problem then is that when the offsets reset on purpose (and not because of a bug), either due to a migration or a Kafka failure that required recreating topics or topic partitions, Retriever detects the unexpected values and <em>refuses to process data</em> to avoid corrupting its content.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">This, in turn, means that Retriever goes down hard and becomes unavailable for querying or new updates. If this happens on one partition, marking it read-only makes the rest of the cluster handle its burdens. If this happens on all partitions, no traffic can make it out of Kafka and to Retriever.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">If Retriever cannot process events within a few days (as configured by the Kafka cluster’s retention), then the messages are eventually pruned and the data is irreversibly lost.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Reducing Retriever’s dependence on offsets can either take one of two forms:</p><ol data-testid="article-list-number" class="text-hc-slate not-prose font-roboto mt-6 list-decimal pl-6 text-sm leading-normal font-normal tracking-normal lg:text-base"><li class="mb-2">make it process offsets <em>and</em> timestamps, such that if the offset goes back but timestamps are greater, we know the partition has reset; otherwise, follow offsets</li><li class="mb-2">have Retriever able to reset the offset counts it tracks back to zero such that it will safely keep processing messages, but without overwriting old ones it has on disk or references in MySQL</li></ol><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Both represent a significant risk to implement, because any bug can irreversibly damage and corrupt data Retriever has stored. In this incident, the evacuation work ended up implementing the second solution as a one-time operation.</p><h3 class="font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24">What is tiered storage?</h3><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Our system sees a lot of data. Tiered storage is a closed-source commercial feature of Confluent that lets Kafka split its storage into two tiers: a local one (hot) and a remote one on S3 (cold). New and fresh data that will be streamed to most live consumers is read from the hot storage, whereas older data (up to a few days old) is stored in S3.</p><img data-testid="article-image" alt="Tiered Storage" fetchPriority="low" loading="lazy" width="897" height="430" decoding="async" data-nimg="1" class="not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full" style="color:transparent" sizes="100vw" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8f9496198c2d829cce17f30acf4092ce357a7fa2-897x430.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8f9496198c2d829cce17f30acf4092ce357a7fa2-897x430.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8f9496198c2d829cce17f30acf4092ce357a7fa2-897x430.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8f9496198c2d829cce17f30acf4092ce357a7fa2-897x430.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8f9496198c2d829cce17f30acf4092ce357a7fa2-897x430.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8f9496198c2d829cce17f30acf4092ce357a7fa2-897x430.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8f9496198c2d829cce17f30acf4092ce357a7fa2-897x430.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8f9496198c2d829cce17f30acf4092ce357a7fa2-897x430.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8f9496198c2d829cce17f30acf4092ce357a7fa2-897x430.png&w=3840&q=75"/><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">This lets us have fewer hosts with fast access for live streaming, and elastic (but slower) storage for occasionally recovering older data.</p><h3 class="font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24">How did Kafka partitions break down?</h3><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">At a steady state, our Kafka clusters in production have various brokers, over three availability zones, with each topic partition replicated across one broker per availability zone (AZ). Here’s an example of what this looks like with six brokers:</p><img data-testid="article-image" alt="Steady State" fetchPriority="low" loading="lazy" width="994" height="790" decoding="async" data-nimg="1" class="not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full" style="color:transparent" sizes="100vw" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff4c03d079c49e23ce0b49c477c36f99d71bc41c2-994x790.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff4c03d079c49e23ce0b49c477c36f99d71bc41c2-994x790.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff4c03d079c49e23ce0b49c477c36f99d71bc41c2-994x790.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff4c03d079c49e23ce0b49c477c36f99d71bc41c2-994x790.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff4c03d079c49e23ce0b49c477c36f99d71bc41c2-994x790.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff4c03d079c49e23ce0b49c477c36f99d71bc41c2-994x790.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff4c03d079c49e23ce0b49c477c36f99d71bc41c2-994x790.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff4c03d079c49e23ce0b49c477c36f99d71bc41c2-994x790.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff4c03d079c49e23ce0b49c477c36f99d71bc41c2-994x790.png&w=3840&q=75"/><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">In this image, you can see five topic partitions (green, blue, yellow, red, and purple), each of which touch three brokers and span three AZs. If you’re a producer of the green topic, you write to the leader (in the leftmost AZ), but can consume from any of them.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">When an AZ fails, all brokers in that AZ die and vanish. The auto-scaling group (ASG) in AWS detects that we only have four out of six nodes, and boots two new ones in functional AZs. Once they’re up, Kafka can start replicating data there:</p><img data-testid="article-image" alt="AZ Failure" fetchPriority="low" loading="lazy" width="1018" height="849" decoding="async" data-nimg="1" class="not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full" style="color:transparent" sizes="100vw" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F704dda8b35ca3597ec26cdd32116d68742b5d6f6-1018x849.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F704dda8b35ca3597ec26cdd32116d68742b5d6f6-1018x849.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F704dda8b35ca3597ec26cdd32116d68742b5d6f6-1018x849.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F704dda8b35ca3597ec26cdd32116d68742b5d6f6-1018x849.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F704dda8b35ca3597ec26cdd32116d68742b5d6f6-1018x849.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F704dda8b35ca3597ec26cdd32116d68742b5d6f6-1018x849.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F704dda8b35ca3597ec26cdd32116d68742b5d6f6-1018x849.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F704dda8b35ca3597ec26cdd32116d68742b5d6f6-1018x849.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F704dda8b35ca3597ec26cdd32116d68742b5d6f6-1018x849.png&w=3840&q=75"/><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">As seen here, two grayed out hosts have been added, and for a while, data will be replicated from in-sync replicas (ISRs) to those out of sync, represented by dotted lines. During that time, consumers can move to any ISR to get up-to-date data and be fine. These ISRs may be a bit more stressed, but overall, things stay up.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">When we killed hosts in the cleanup, <em>older</em> brokers accidentally got killed. This can further reduce redundancy, but in some unlucky cases, may fully remove all brokers that owned a topic partition:</p><img data-testid="article-image" alt="Bad Recovery" fetchPriority="low" loading="lazy" width="925" height="586" decoding="async" data-nimg="1" class="not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full" style="color:transparent" sizes="100vw" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fb0dcbcb86dc41665fd9e40739f5e24a309cbe86b-925x586.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fb0dcbcb86dc41665fd9e40739f5e24a309cbe86b-925x586.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fb0dcbcb86dc41665fd9e40739f5e24a309cbe86b-925x586.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fb0dcbcb86dc41665fd9e40739f5e24a309cbe86b-925x586.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fb0dcbcb86dc41665fd9e40739f5e24a309cbe86b-925x586.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fb0dcbcb86dc41665fd9e40739f5e24a309cbe86b-925x586.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fb0dcbcb86dc41665fd9e40739f5e24a309cbe86b-925x586.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fb0dcbcb86dc41665fd9e40739f5e24a309cbe86b-925x586.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fb0dcbcb86dc41665fd9e40739f5e24a309cbe86b-925x586.png&w=3840&q=75"/><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">In this image, you can see that we killed the top two brokers (in pale red) which were in sync. Only two out of six old brokers remain, and only they have in sync data. You can see that topic partitions yellow and purple have only one healthy broker (middle-left for yellow, middle-right for purple), red was on both middle brokers and is fine, but green and blue were on neither of the remaining hosts.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">In these cases, red and blue would find themselves leaderless. The only thing telling how much data they could find would be to know how far behind the new greyed out brokers were, and whether critical metadata had the time to be replicated.</p><h3 class="font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24">Why did it only happen in prod?</h3><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The key lies in the topology of the clusters. While the production cluster has many nodes, our pre-production clusters in the EU only have fewer nodes, in a way that makes this far less likely.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">Here’s an example with the smallest cluster possible, with only three nodes. When an AZ failure happens there, only one node gets killed, and once again, all topic partitions have two out of three brokers present:</p><img data-testid="article-image" alt="AZ Failure" fetchPriority="low" loading="lazy" width="1013" height="780" decoding="async" data-nimg="1" class="not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full" style="color:transparent" sizes="100vw" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff20adc6769e7ba2df764868d3907c3435a8f166b-1013x780.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff20adc6769e7ba2df764868d3907c3435a8f166b-1013x780.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff20adc6769e7ba2df764868d3907c3435a8f166b-1013x780.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff20adc6769e7ba2df764868d3907c3435a8f166b-1013x780.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff20adc6769e7ba2df764868d3907c3435a8f166b-1013x780.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff20adc6769e7ba2df764868d3907c3435a8f166b-1013x780.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff20adc6769e7ba2df764868d3907c3435a8f166b-1013x780.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff20adc6769e7ba2df764868d3907c3435a8f166b-1013x780.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Ff20adc6769e7ba2df764868d3907c3435a8f166b-1013x780.png&w=3840&q=75"/><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">The key difference happens because we only killed one host during the failure test. Since only one was killed during the test, only one needs to be killed as part of the clean up (either of the black-outlined nodes above). By killing a single broker, we hit a total of two, and it follows that no partition will go below a single broker left, which will remain a safe leader.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">This means that so long as you start the AZ failure with a healthy cluster, it is difficult to break the cluster. We ran four tests in pre-production clusters, but since the issue was structurally unlikely to be reproduced there, the flaw remained hidden. On top of all the other risks of doing an AZ failover like this (database failovers, large amounts of connection interruptions across multiple services, etc.) and our frequent chaos engineering around Kafka, this made everything seem adequate on that front. With limited time and resources, other aspects of the exercise that seemed riskier were what drew the attention of people running it.</p><p class="text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24">While running multiple tests in a cluster more similar to production might still not have caught the issue, the likelihood would have been much higher.</p></div></div><aside id="relatedArticles" class="col-span-full py-12 lg:py-24"><div class="z-20 flex flex-col items-center lg:w-1/2 lg:gap-8 mx-auto mb-0 gap-8 text-center lg:mb-0"><h1 _type="title" class="font-medium text-pretty font-poppins text-center text-4xl lg:text-5xl"><span class="title-hero-heading text-4xl tracking-tighter lg:text-6xl"></span></h1></div><div class="gap-4 py-10 lg:py-16 xl:flex xl:gap-6"><div class="full flex flex-col gap-4 md:flex-row lg:flex-col xl:w-[831px] xl:gap-6"><a class="flex-1 lg:h-[245px]" href="/resources/podcasts/ep-93-adaptive-sampling-strategies-with-mike-goldsmith"><div class="bg-hc-gray-50 flex h-full flex-col gap-6 rounded-4xl p-2.5 lg:flex-row"><div class="relative h-[172px] w-auto md:h-[177px] lg:h-[225px] lg:w-[396px] xl:flex-1"><img alt="O11yCast podcast logo: a dark background with a radiating orange, blue, green, and yellow circular pattern above the white and orange text '011Y_Cast(((' and 'Everything Observability'." loading="lazy" decoding="async" data-nimg="fill" class="aspect-video rounded-4xl object-cover" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" sizes="(max-width: 768px) 100vw, 396px" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png&w=3840&q=75"/></div><div class="flex flex-1 flex-col gap-4 max-lg:px-2.5 lg:pt-4"><div class="flex items-center gap-4"><div class="font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-sky-500 text-sky-50">Podcasts</div><span class="h-3.5 border-r border-r-black/30"></span><span class="font-medium font-roboto text-hc-slate/70 text-xs"><span>September 3, 2026</span></span></div><h3 class="font-roboto line-clamp-2 text-base font-medium">Ep. #93, Adaptive Sampling Strategies with Mike Goldsmith</h3><p class="font-roboto text-hc-slate/70 line-clamp-3 text-sm font-normal max-lg:hidden">On episode 93 of O11ycast, Mike Goldsmith talks to Martin Thwaites and Ken Rimple discuss Honeycomb's contribution to the OpenTelemetry Collector, the Adaptive Tail Sampling Processor. This processor was created based on years of experience building Honeycomb's custom, industry leading tail sampler, Refinery.
|
||
|
||
</p></div></div></a><a class="flex-1 lg:h-[245px]" href="/resources/leading-with-observability/leading-with-observability-scaling-fin-to-2x-engineering-productivity"><div class="bg-hc-gray-50 flex h-full flex-col gap-6 rounded-4xl p-2.5 lg:flex-row"><div class="relative h-[172px] w-auto md:h-[177px] lg:h-[225px] lg:w-[396px] xl:flex-1"><img alt="Title slide for the series "Leading With Observability," Episode 1: "Scaling Fin to 2x Engineering Productivity," featuring Darragh Curran (CTO, Fin) and Charity Majors (CTO, Honeycomb)." loading="lazy" decoding="async" data-nimg="fill" class="aspect-video rounded-4xl object-cover" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" sizes="(max-width: 768px) 100vw, 396px" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg&w=3840&q=75"/></div><div class="flex flex-1 flex-col gap-4 max-lg:px-2.5 lg:pt-4"><div class="flex items-center gap-4"><div class="font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-gray-200 text-hc-slate">Leading With Observability</div><span class="h-3.5 border-r border-r-black/30"></span><span class="font-medium font-roboto text-hc-slate/70 text-xs"><span>August 27, 2026</span></span></div><h3 class="font-roboto line-clamp-2 text-base font-medium">Leading With Observability: Scaling Fin to 2x Engineering Productivity</h3><p class="font-roboto text-hc-slate/70 line-clamp-3 text-sm font-normal max-lg:hidden">Darragh Curran, CTO at Fin (formerly Intercom), set out to double engineering productivity—and nearly tripled it instead, using AI-driven code, an AI-powered PR review system, observability as a trust mechanism, and much more hands-on leadership. In the first episode of Leading With Observability, Charity gets into the messy bits behind that transformation, not just the highlight reel.</p></div></div></a></div><div class="full flex flex-col gap-4 max-xl:mt-4 md:flex-col lg:flex-row xl:flex-1 xl:flex-col xl:gap-6"><a class="flex-1 lg:h-[130px]" href="/resources/webinars/aws-honeycomb-workshop-ai-investigators"><div class="bg-hc-gray-50 flex h-full gap-4 rounded-3xl p-4 xl:gap-6 xl:rounded-4xl xl:p-2.5"><div class="relative h-[84px] w-[84px] self-start xl:h-[135px] xl:w-[135px]"><div class="aspect-square h-full w-full overflow-hidden rounded-2xl bg-white xl:rounded-3xl"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 542 542" fill="none" class="h-full w-full"><path fill="#FBEDF0" d="M0 0h542v542H0z"></path><path fill="#F7D2D7" d="M460.213 394.786c38.019-55.443 16.608-179.375-78.922-214.08-56.823-20.676-114.265-101.902-218.578-66.827a195.6 195.6 0 0 0-8.51 3.138 160.962 160.962 0 0 0-8.235 3.692C38.36 174.244 36.576 301.622 141.37 345.065c89.764 37.168 248.431 152.484 318.843 49.721Z"></path><rect width="318" height="202" x="107" y="166" fill="#fff" rx="5"></rect><rect width="318" height="202" x="107" y="166" stroke="#E4A3A5" stroke-width="10" rx="5"></rect><circle cx="198" cy="243" r="23" fill="#E65B53"></circle><path fill="#E65B53" d="M155 314a43.001 43.001 0 0 1 73.406-30.406A43.001 43.001 0 0 1 241 314h-86Z"></path><g clip-path="url(#a)"><path fill="#FBEDF0" d="M301 187h47.472v47.472H301z"></path><circle cx="324.737" cy="202.844" r="7.053" fill="#E65B53"></circle><path fill="#E65B53" d="M311.55 224.618a13.19 13.19 0 0 1 13.187-13.187 13.188 13.188 0 0 1 13.187 13.187H311.55Z"></path></g><g clip-path="url(#b)"><path fill="#FBEDF0" d="M301 243.264h47.472v47.472H301z"></path><circle cx="324.737" cy="259.108" r="7.053" fill="#E65B53"></circle><path fill="#E65B53" d="M311.55 280.882a13.188 13.188 0 0 1 26.374 0H311.55Z"></path></g><g clip-path="url(#c)"><path fill="#FBEDF0" d="M301 299.527h47.472v47.472H301z"></path><circle cx="324.737" cy="315.372" r="7.053" fill="#E65B53"></circle><path fill="#E65B53" d="M311.55 337.145a13.187 13.187 0 0 1 26.374 0H311.55Z"></path></g><g clip-path="url(#d)"><path fill="#FBEDF0" d="M357.264 187h47.472v47.472h-47.472z"></path><circle cx="381.001" cy="202.844" r="7.053" fill="#E65B53"></circle><path fill="#E65B53" d="M367.814 224.618a13.188 13.188 0 0 1 13.187-13.187 13.186 13.186 0 0 1 13.187 13.187h-26.374Z"></path></g><g clip-path="url(#e)"><path fill="#FBEDF0" d="M357.264 243.264h47.472v47.472h-47.472z"></path><circle cx="381.001" cy="259.108" r="7.053" fill="#E65B53"></circle><path fill="#E65B53" d="M367.814 280.882a13.187 13.187 0 1 1 26.374 0h-26.374Z"></path></g><g clip-path="url(#f)"><path fill="#FBEDF0" d="M357.264 299.527h47.472v47.472h-47.472z"></path><circle cx="381.001" cy="315.372" r="7.053" fill="#E65B53"></circle><path fill="#E65B53" d="M367.814 337.145a13.186 13.186 0 1 1 26.374 0h-26.374Z"></path></g><circle cx="412.5" cy="188.5" r="53.5" fill="#E65B53"></circle><path fill="#fff" d="M434.842 186.768c1.333.77 1.333 2.695 0 3.464l-32.013 18.483c-1.334.77-3-.192-3-1.732v-36.965c0-1.54 1.666-2.502 3-1.733l32.013 18.483Z"></path><path fill="#D75450" d="M238 373h55v27h-55z"></path><rect width="168" height="27" x="182" y="400" fill="#E4A3A5" rx="13.5"></rect><path stroke="#D75450" stroke-linecap="round" stroke-width="5" d="M87.805 426.408c5.565 2.626 19.049 5.505 28.461-3.987 9.412-9.491 8.73-19.101 2.011-24.32-5.871-4.135-17.918-1.146-17.026 10.748 1.116 14.868 23.483 16.543 40.084 5.253"></path><defs><clipPath id="a"><path fill="#fff" d="M301 187h47.472v47.472H301z"></path></clipPath><clipPath id="b"><path fill="#fff" d="M301 243.264h47.472v47.472H301z"></path></clipPath><clipPath id="c"><path fill="#fff" d="M301 299.527h47.472v47.472H301z"></path></clipPath><clipPath id="d"><path fill="#fff" d="M357.264 187h47.472v47.472h-47.472z"></path></clipPath><clipPath id="e"><path fill="#fff" d="M357.264 243.264h47.472v47.472h-47.472z"></path></clipPath><clipPath id="f"><path fill="#fff" d="M357.264 299.527h47.472v47.472h-47.472z"></path></clipPath></defs></svg></div></div><div class="flex flex-1 flex-col items-start gap-3 xl:pt-[6px]"><div class="font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-purple-100 text-purple-800">Webinars</div><h4 class="font-roboto line-clamp-3 text-sm font-medium xl:text-base">How Observability Makes Better AI and Human Investigators</h4></div></div></a><a class="flex-1 lg:h-[130px]" href="/resources/getting-started/what-is-contextual-observability"><div class="bg-hc-gray-50 flex h-full gap-4 rounded-3xl p-4 xl:gap-6 xl:rounded-4xl xl:p-2.5"><div class="relative h-[84px] w-[84px] self-start xl:h-[135px] xl:w-[135px]"><div class="aspect-square h-full w-full overflow-hidden rounded-2xl bg-white xl:rounded-3xl"><img alt="What Is Contextual Observability? A Practitioner's Guide for Modern Engineering Teams" loading="lazy" decoding="async" data-nimg="fill" class="aspect-square rounded-[14px] object-cover xl:rounded-4xl" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" sizes="(max-width: 768px) 25vw, 135px" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=256&q=75 256w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=384&q=75 384w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2Fa01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png&w=3840&q=75"/></div></div><div class="flex flex-1 flex-col items-start gap-3 xl:pt-[6px]"><div class="font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-green-800 text-hc-green-50">Getting Started</div><h4 class="font-roboto line-clamp-3 text-sm font-medium xl:text-base">What Is Contextual Observability? A Practitioner's Guide for Modern Engineering Teams</h4></div></div></a><a class="flex-1 lg:h-[130px]" href="/resources/conference-talks/socratic-ai-integrating-observability-through-interactive-dialogue-o11ycon-2026"><div class="bg-hc-gray-50 flex h-full gap-4 rounded-3xl p-4 xl:gap-6 xl:rounded-4xl xl:p-2.5"><div class="relative h-[84px] w-[84px] self-start xl:h-[135px] xl:w-[135px]"><div class="aspect-square h-full w-full overflow-hidden rounded-2xl bg-white xl:rounded-3xl"><img alt="Socratic AI: Integrating Observability Through Interactive Dialogue - O11yCon 2026" loading="lazy" decoding="async" data-nimg="fill" class="aspect-square rounded-[14px] object-cover xl:rounded-4xl" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" sizes="(max-width: 768px) 25vw, 135px" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=256&q=75 256w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=384&q=75 384w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=640&q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=750&q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=828&q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=1080&q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=1200&q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=1920&q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=2048&q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=3840&q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2F927dxq0h%2Fproduction%2F8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png&w=3840&q=75"/></div></div><div class="flex flex-1 flex-col items-start gap-3 xl:pt-[6px]"><div class="font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-gold-300 text-gold-1000">Conference Talks</div><h4 class="font-roboto line-clamp-3 text-sm font-medium xl:text-base">Socratic AI: Integrating Observability Through Interactive Dialogue - O11yCon 2026</h4></div></div></a></div></div></aside></article></main><!--$--><!--/$--><footer class="original-footer font-roboto text-hc-slate mx-auto max-w-[1152px] px-4"><div class="flex w-full flex-col items-center gap-[40px] py-[80px] lg:flex-row lg:items-stretch lg:gap-[132px]"><div class="order-2 flex-1 lg:order-1"><div class="hidden flex-1 flex-row justify-between pb-[74px] lg:flex"><div class="flex flex-col gap-3"><span class="text-base leading-[150%] font-semibold">Observability Platform</span><ul><li class="pb-1"><a target="_self" _key="3709d74ab43c" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/pricing">Pricing</a></li><li class="pb-1"><a target="_self" _key="c73c8241288b" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform">Platform Overview</a></li><li class="pb-1"><a target="_self" _key="7270c5c87c85" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/frontend-observability">Frontend Observability</a></li><li class="pb-1"><a target="_self" _key="db25504b1c78" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/log-analytics">Log Analytics</a></li><li class="pb-1"><a target="_self" _key="ea87e5e00146" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/distributed-tracing">Distributed Tracing</a></li><li class="pb-1"><a target="_self" _key="045ae34eb65d" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/metrics">Metrics</a></li><li class="pb-1"><a target="_self" _key="3d1692a51e34" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/telemetry-pipeline">Telemetry Pipeline</a></li><li class="pb-1"><a target="_self" _key="4196f597a92d" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/private-cloud">Private Cloud</a></li><li class="pb-1"><a target="_self" _key="b22d2cde4a3d" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/slos">SLOs</a></li><li class="pb-1"><a target="_self" _key="0f5a9a5f7aa7" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/service-map">Service Map</a></li><li class="pb-1"><a target="_self" _key="6db0fbacbbc4" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/bubbleup">BubbleUp</a></li><li class="pb-1"><a target="_self" _key="d672da05edda" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/opentelemetry">OpenTelemetry</a></li><li class="pb-1"><a target="_self" _key="80b1b84cfd32" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/platform/integrations">App Integrations</a></li></ul></div><div class="flex flex-col gap-3"><span class="text-base leading-[150%] font-semibold">Solutions</span><ul><li class="pb-1"><a target="_self" _key="8f77d4ed6602" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/why-honeycomb">Why Honeycomb</a></li><li class="pb-1"><a target="_self" _key="7fb03607561c" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/why-honeycomb/enterprise">Enterprise</a></li><li class="pb-1"><a target="_self" _key="d0d6163b27ff" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/technologies/ai-agents">AI Agents</a></li><li class="pb-1"><a target="_self" _key="2b90f65f46e6" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/technologies/aws">Amazon Web Services</a></li><li class="pb-1"><a target="_self" _key="c3dff0996d2b" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/technologies/microsoft-azure">Microsoft Azure</a></li><li class="pb-1"><a target="_self" _key="d0caa1e4eb66" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/technologies/kubernetes">Kubernetes</a></li><li class="pb-1"><a target="_self" _key="9ceaefb31039" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/technologies/google-cloud">Google Cloud</a></li></ul></div><div class="flex flex-col gap-3"><span class="text-base leading-[150%] font-semibold">Company</span><ul><li class="pb-1"><a target="_self" _key="26df551d2e36" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/about">About Us</a></li><li class="pb-1"><a target="_self" _key="cd5a23b41429" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/about/team">Team</a></li><li class="pb-1"><a target="_self" _key="9e8d0ae5c80c" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/careers">Careers</a></li><li class="pb-1"><a target="_self" _key="21e1fb267d5e" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/news">News</a></li><li class="pb-1"><a target="_self" _key="bb5ab8caffa9" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/events">Events</a></li><li class="pb-1"><a target="_self" _key="f8a10f88008f" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/why-honeycomb/customers">Customers</a></li><li class="pb-1"><a target="_self" _key="bf0deb38bf75" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/partners">Partners</a></li><li class="pb-1"><a target="_blank" rel="noopener noreferrer" _key="15f6dabb07e6" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="https://docs.honeycomb.io/security-compliance">Security</a></li><li class="pb-1"><a target="_self" _key="85057eb711e5" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/support">Support</a></li><li class="pb-1"><a target="_blank" rel="noopener noreferrer" _key="e94d32b2d070" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="https://status.honeycomb.io/">Status</a></li><li class="pb-1"><a target="_self" _key="0ea03725759b" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="https://changelog.honeycomb.io/">Changelog</a></li></ul></div><div class="flex flex-col gap-3"><span class="text-base leading-[150%] font-semibold">Resources</span><ul><li class="pb-1"><a target="_self" _key="671f18da9b3f" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/blog">Blog</a></li><li class="pb-1"><a target="_self" _key="92ccb060bb8f" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="https://docs.honeycomb.io">Documentation</a></li><li class="pb-1"><a target="_self" _key="db87d691804a" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/resources/getting-started">Getting Started</a></li><li class="pb-1"><a target="_self" _key="ba793c2d2aec" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/resources/case-studies">Case Studies</a></li><li class="pb-1"><a target="_self" _key="b6324b7fa3ad" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/why-honeycomb/comparisons">Comparisons</a></li><li class="pb-1"><a target="_self" _key="8dacd4366e92" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/resources/webinars">Webinars</a></li><li class="pb-1"><a target="_self" _key="2e34173dd022" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/blog/what-is-observability-key-components-best-practices">What Is Observability?</a></li><li class="pb-1"><a target="_self" _key="52309d8a3d42" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/resources/getting-started/what-is-ai-observability">What Is AI Observability?</a></li><li class="pb-1"><a target="_self" _key="a1a79caf1ea6" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/resources/getting-started/what-is-llm-observability">What Is LLM Observability?</a></li><li class="pb-1"><a target="_self" _key="9cf8c6d40ac4" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/resources/getting-started/application-performance-monitoring-vs-observability">APM vs Observability</a></li><li class="pb-1"><a target="_self" _key="b20174d4157b" _type="link" class="text-sm leading-[20px] tracking-[0.02em]" href="/resources/getting-started/getting-started-with-opentelemetry">OpenTelemetry Guide</a></li></ul></div></div><div class="flex flex-col items-baseline gap-2 text-xs leading-[150%] tracking-[0] lg:flex-row lg:gap-3"><span>© <!-- -->2026<!-- --> <!-- -->Hound Technology, Inc.</span><ul class="flex flex-row gap-3"><li class="pb-1"><a target="_self" _key="4042940388e7" _type="link" class="text-hc-cobalt text-xs leading-[20px] tracking-[0.02em] underline" href="/terms">Terms of Service</a></li><li class="pb-1"><a target="_self" _key="df3d0de1abf9" _type="link" class="text-hc-cobalt text-xs leading-[20px] tracking-[0.02em] underline" href="/acceptable-use-policy">Acceptable Use Policy</a></li><li class="pb-1"><a target="_self" _key="de1418f17f9f" _type="link" class="text-hc-cobalt text-xs leading-[20px] tracking-[0.02em] underline" href="/privacy">Privacy Notice</a></li><li class="pb-1"><a href="#" class="text-hc-cobalt cky-banner-element text-xs leading-[20px] tracking-[0.02em] underline">Your Privacy Choices</a></li></ul></div></div><div class="order-1 flex w-full flex-grow flex-col items-center justify-between self-stretch lg:order-2 lg:max-w-[295px]"><div id="footer-newsletter" class="flex-1 lg:px-4"><p class="font-roboto pb-[15px] text-center text-lg leading-[22px] font-bold tracking-[0.02em]">Sign up for updates from Honeycomb</p><div id="footer-form" data-form-id="6ad8ab28-e5df-4a57-a1eb-b5fad2435f8d" class="hbspt-form [&_.submitted-message]:!block [&_.submitted-message_p]:my-4 [&_.submitted-message_p]:text-center [&_.submitted-message_p]:font-bold [&_.submitted-message_p]:tracking-wide"></div></div><div class="mt-auto pt-4"><div class="mt-4 flex gap-4"><a href="https://bsky.app/profile/honeycomb.io" target="_blank" rel="noopener noreferrer" aria-label="Bluesky" class="transition-opacity hover:opacity-80"><div class="border-hc-slate flex size-9 items-center justify-center rounded-full border-[2px]"><img alt="Bluesky" loading="lazy" width="600" height="530" decoding="async" data-nimg="1" class="h-3.5 w-3.5 object-contain" style="color:transparent" src="https://cdn.sanity.io/images/927dxq0h/production/b791a32106224d6a8c17908d9e10a1098ecd033e-600x530.svg"/></div></a><a href="https://www.linkedin.com/company/honeycomb.io" target="_blank" rel="noopener noreferrer" aria-label="linkedin" class="transition-opacity hover:opacity-80"><div class="border-hc-slate flex size-9 items-center justify-center rounded-full border-[2px]"><img alt="Linkedin" loading="lazy" width="36" height="36" decoding="async" data-nimg="1" class="h-3.5 w-3.5 object-contain" style="color:transparent" src="https://cdn.sanity.io/images/927dxq0h/production/2129b6ff6ad7d0a679bcde4a6d1f3498d2c5c2c3-36x36.svg"/></div></a><a href="https://www.youtube.com/channel/UCty8KGQ3oAP0MQQmLIv7k0Q" target="_blank" rel="noopener noreferrer" aria-label="youtube" class="transition-opacity hover:opacity-80"><div class="border-hc-slate flex size-9 items-center justify-center rounded-full border-[2px]"><img alt="YouTube" loading="lazy" width="256" height="256" decoding="async" data-nimg="1" class="h-3.5 w-3.5 object-contain" style="color:transparent" src="https://cdn.sanity.io/images/927dxq0h/production/6949a9f5fcdca86c2505079300c317729031c030-256x256.svg"/></div></a><a href="https://www.honeycomb.io/rss/blog.xml" target="_blank" rel="noopener noreferrer" aria-label="RSS" class="transition-opacity hover:opacity-80"><div class="border-hc-slate flex size-9 items-center justify-center rounded-full border-[2px]"><img alt="RSS" loading="lazy" width="800" height="800" decoding="async" data-nimg="1" class="h-3.5 w-3.5 object-contain" style="color:transparent" src="https://cdn.sanity.io/images/927dxq0h/production/c0e681e6a520b9ddc9b0beb072859fd9b6e7e20e-800x800.svg"/></div></a></div></div></div></div></footer><footer class="font-roboto text-hc-slate small-footer container hidden flex-col items-center justify-between gap-10 py-10 lg:flex-row lg:gap-5 xl:gap-0"><ul class="flex flex-1 flex-col justify-between gap-5 text-center lg:flex-2/3 lg:flex-row lg:gap-0 lg:text-left"><li><a class="text-hc-cobalt leading-[20px] tracking-[0.02em] underline" href="https://honeycomb.io">Honeycomb.io</a></li><li><a target="_self" _key="4042940388e7" _type="link" class="text-hc-cobalt leading-[20px] tracking-[0.02em] text-nowrap underline" href="/terms">Terms of Service</a></li><li><a target="_self" _key="df3d0de1abf9" _type="link" class="text-hc-cobalt leading-[20px] tracking-[0.02em] text-nowrap underline" href="/acceptable-use-policy">Acceptable Use Policy</a></li><li><a target="_self" _key="de1418f17f9f" _type="link" class="text-hc-cobalt leading-[20px] tracking-[0.02em] text-nowrap underline" href="/privacy">Privacy Notice</a></li><li><a class="cky-banner-element text-hc-cobalt leading-[20px] tracking-[0.02em] text-nowrap underline" href="#">Your Privacy Choices</a></li></ul><span class="text-end leading-[20px] tracking-[0.02em] lg:max-w-fit xl:max-w-none xl:flex-1/3">© <!-- -->2026<!-- --> <!-- -->Hound Technology, Inc.</span></footer><script src="/_next/static/chunks/05gogik_6g8vk.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[516503,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\"],\"\"]\n3:I[483136,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\"],\"default\"]\n4:I[758298,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\",\"/_next/static/chunks/12~zlyzq33k~1.js\"],\"default\"]\n5:I[229390,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\"],\"default\"]\n11:I[563491,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\",\"/_next/static/chunks/025lo2k4b20~m.js\"],\"default\"]\n:HL[\"/_next/static/chunks/0-a~7ljgpwyij.css\",\"style\"]\n:HL[\"/_next/static/chunks/0k9no2apwmz~n.css\",\"style\"]\n:HL[\"/_next/static/chunks/0nuksys438q2t.css\",\"style\"]\n:HL[\"/_next/static/media/345b029bff63acdd-s.p.14zxo1jbez.lc.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/8e6fa89aa22d24ec-s.p.0uvzar8hswo3p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/ce62453a442c7f35-s.p.0333ktddfbsxy.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"/_next/static/media/e2334d715941921e-s.p.12skym0rqknxy.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"c\":[\"\",\"blog\",\"incident-report-exercises-cleanups-and-evacuations\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"(frontend)\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"incident-report-exercises-cleanups-and-evacuations\",\"d\",null],{\"children\":[\"__PAGE__\",{}]}]}]}]},\"$undefined\",\"$undefined\",16],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/0-a~7ljgpwyij.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/0k9no2apwmz~n.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/chunks/019vi3l~x0zoc.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[[\"$\",\"head\",null,{\"children\":[[[\"$\",\"link\",null,{\"rel\":\"preconnect\",\"href\":\"https://cdn-cookieyes.com\"}],[\"$\",\"link\",null,{\"rel\":\"dns-prefetch\",\"href\":\"https://cdn-cookieyes.com\"}],[\"$\",\"$L2\",null,{\"id\":\"cookieyes-preserve-native\",\"strategy\":\"beforeInteractive\",\"children\":\"window.ckySettings=window.ckySettings||{};window.ckySettings.nativeFunctions={fetch:window.fetch,xhr:window.XMLHttpRequest};\"}],[\"$\",\"$L2\",null,{\"id\":\"cookieyes\",\"src\":\"https://cdn-cookieyes.com/client_data/497224d8995aa8565cbc3037/script.js\",\"strategy\":\"beforeInteractive\"}]],[\"$\",\"script\",null,{\"referrerPolicy\":\"no-referrer-when-downgrade\",\"src\":\"https://edge.wingify.net/tag/1272784.js\"}],[[\"$\",\"$L2\",null,{\"src\":\"/hny.min.js\",\"strategy\":\"beforeInteractive\"}],[\"$\",\"$L2\",null,{\"id\":\"hny-init\",\"strategy\":\"beforeInteractive\",\"children\":\"\\n window.Hny.initializeTracing({\\n debug: false,\\n apiKey: \\\"hcaik_01kc76rhfsq3kjf9tdmx6f7bmhhd0rzcdrpbtbgk31bpj4nezy2frmbqx8\\\",\\n serviceName: \\\"www-next\\\",\\n configDefaults: {\\n ignoreNetworkEvents: true,\\n propagateTraceHeaderCorsUrls: [\\n /https:\\\\/\\\\/www\\\\.honeycomb\\\\.io/,\\n /https:\\\\/\\\\/www-stage\\\\.honeycomb\\\\.io/,\\n /https:\\\\/\\\\/honeycomb\\\\.vercel\\\\.app/,\\n /https:\\\\/\\\\/honeycomb-stage\\\\.vercel\\\\.app/\\n ]\\n }\\n });\\n \"}]],[\"$\",\"$L2\",null,{\"src\":\"https://stats.honeycomb.io/js/pa-DJvf3AtvU5wDjKkEU-JVs.js\",\"strategy\":\"beforeInteractive\",\"async\":true,\"id\":\"plausible-script\"}],[\"$\",\"$L2\",null,{\"id\":\"plausible-init\",\"strategy\":\"beforeInteractive\",\"children\":\"\\n window.plausible = window.plausible || function() {\\n (plausible.q = plausible.q || []).push(arguments)\\n };\\n plausible.init = plausible.init || function(i) {\\n plausible.o = i || {}\\n };\\n plausible.init();\\n \"}]]}],[\"$\",\"body\",null,{\"className\":\"roboto_a526148e-module__zhi6Uq__className poppins_b750422f-module__FR8AEW__className roboto_459e8801-module__Bzopaa__className poppins_c92f7652-module__gE1l-G__className\",\"children\":[[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$4\",\"errorStyles\":[],\"errorScripts\":[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/12~zlyzq33k~1.js\",\"async\":true}]],\"template\":[\"$\",\"$L5\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],\"$L6\",\"$L7\"]}]}]],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}],\"$L8\",\"$L9\",\"$La\"]}]]}]]}],{\"children\":[\"$Lb\",{\"children\":[\"$Lc\",{\"children\":[\"$Ld\",{\"children\":[\"$Le\",{},null,false,null]},null,false,\"$@f\"]},null,false,\"$@f\"]},null,false,null]},null,false,null],\"$L10\",false]],\"m\":\"$undefined\",\"G\":[\"$11\",[\"$L12\",\"$L13\"]],\"S\":true,\"h\":null,\"s\":\"$undefined\",\"l\":\"$undefined\",\"p\":\"$undefined\",\"d\":\"$undefined\",\"b\":\"VVuS_Z0Yp8bYu3V6Xy5hL\"}\n"])</script><script>self.__next_f.push([1,"14:I[694113,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\"],\"GoogleAnalytics\"]\n15:I[196641,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\"],\"GoogleTagManager\"]\n18:I[351397,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\"],\"OutletBoundary\"]\n19:\"$Sreact.suspense\"\n1c:I[351397,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\"],\"ViewportBoundary\"]\n1e:I[351397,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\"],\"MetadataBoundary\"]\n6:[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":404}]\n7:[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]\n8:[\"$\",\"$L14\",null,{\"gaId\":\"G-DJM5ZXPGB0\"}]\n9:[\"$\",\"$L15\",null,{\"gtmId\":\"GTM-KH7NL6K\"}]\na:[\"$\",\"$L2\",null,{\"strategy\":\"afterInteractive\",\"id\":\"wphs\",\"src\":\"https://cdn.jsdelivr.net/npm/hockeystack@latest/hockeystack.min.js\",\"async\":true,\"data-apikey\":\"b7867a90ef8ece343a6e29fc6aa01e\",\"data-cookieless\":\"1\",\"data-auto-identify\":\"1\"}]\nb:[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/0nuksys438q2t.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/0ku~nn_l8-fol.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/chunks/039u5_5hlemld.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/chunks/189hrhe.fk4af.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/chunks/0oz-qxk30w~qw.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-4\",{\"src\":\"/_next/static/chunks/0vx_phl8cla2s.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-5\",{\"src\":\"/_next/static/chunks/0xywjwd04osrr.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-6\",{\"src\":\"/_next/static/chunks/0t1~yeqvn891r.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-7\",{\"src\":\"/_next/static/chunks/0ks4m351.jbv2.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-8\",{\"src\":\"/_next/static/chunks/01gy5f4286e6a.js\",\"async\":true,\"nonce\":\"$undefined\"}]],\"$L16\"]}]\nc:[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L5\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}]\nd:[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L5\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}]\ne:[\"$\",\"$1\",\"c\",{\"children\":[\"$L17\",[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/0h3ejemwt0u_c.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"$L18\",null,{\"children\":[\"$\",\"$19\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@1a\"}]}]]}]\n1b:[]\nf:\"$W1b\"\n10:[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$L1c\",null,{\"children\":\"$L1d\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$L1e\",null,{\"children\":[\"$\",\"$19\",null,{\"name\":\"Next.Metadata\",\"children\":\"$L1f\"}]}]}],[\"$\",\"meta\",null,{\"name\":\"next-size-adjust\",\"content\":\"\"}]]}]\n12:[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/0-a~7ljgpwyij.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]\n13:[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/0k9no2apwmz~n.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]\n"])</script><script>self.__next_f.push([1,"1d:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}]]\n"])</script><script>self.__next_f.push([1,"20:I[955297,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\",\"/_next/static/chunks/0ku~nn_l8-fol.js\",\"/_next/static/chunks/039u5_5hlemld.js\",\"/_next/static/chunks/189hrhe.fk4af.js\",\"/_next/static/chunks/0oz-qxk30w~qw.js\",\"/_next/static/chunks/0vx_phl8cla2s.js\",\"/_next/static/chunks/0xywjwd04osrr.js\",\"/_next/static/chunks/0t1~yeqvn891r.js\",\"/_next/static/chunks/0ks4m351.jbv2.js\",\"/_next/static/chunks/01gy5f4286e6a.js\"],\"MobileMenuProvider\"]\n21:I[303231,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\",\"/_next/static/chunks/0ku~nn_l8-fol.js\",\"/_next/static/chunks/039u5_5hlemld.js\",\"/_next/static/chunks/189hrhe.fk4af.js\",\"/_next/static/chunks/0oz-qxk30w~qw.js\",\"/_next/static/chunks/0vx_phl8cla2s.js\",\"/_next/static/chunks/0xywjwd04osrr.js\",\"/_next/static/chunks/0t1~yeqvn891r.js\",\"/_next/static/chunks/0ks4m351.jbv2.js\",\"/_next/static/chunks/01gy5f4286e6a.js\"],\"Header\"]\n"])</script><script>self.__next_f.push([1,"16:[[\"$\",\"$L20\",null,{\"children\":[\"$\",\"$L21\",null,{\"buttons\":[{\"_key\":\"9979c08d8605\",\"_type\":\"button\",\"iconReference\":null,\"link\":{\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":{\"_id\":\"76c12af0-d65b-4e4f-ba3f-c25e9d904ad1\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"get-a-demo\",\"summary\":null,\"title\":\"Get a Honeycomb Demo\"},\"target\":\"_self\",\"text\":\"Get a demo\"},\"modalForm\":null,\"modalVideo\":{\"_type\":\"videoPlayer\",\"uploadedVideo\":null,\"videoSource\":\"upload\"},\"variant\":\"solid\"},{\"_key\":\"f1b0985cac23\",\"_type\":\"button\",\"iconReference\":null,\"link\":{\"_type\":\"link\",\"externalLink\":\"https://ui.honeycomb.io/signup\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Start for free\"},\"modalForm\":null,\"modalVideo\":null,\"variant\":\"outline\"}],\"loginUrl\":\"https://ui.honeycomb.io/login\",\"preHeader\":{\"button\":{\"_type\":\"button\",\"iconReference\":null,\"link\":{\"_type\":\"link\",\"internalLink\":{\"_id\":\"da07f58b-9f3f-4e77-9ebc-140a67c2dc94\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"alt\":\"Observability Engineering O'Reilly Book\",\"asset\":{\"_ref\":\"image-823053e26c243df14b4e682ba2e91d260e06f401-2880x1590-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"observability-engineering-oreilly-book\",\"summary\":null,\"title\":\"Observability Engineering O'Reilly Book\"},\"target\":\"_self\",\"text\":\"Get your copy\"},\"modalForm\":null,\"modalVideo\":null,\"variant\":\"solid-secondary\"},\"text\":\"Observability Engineering second edition out now! 27 net-new chapters written for today's observability challenges.\"},\"sections\":[{\"_key\":\"b97aeefe06e8\",\"_type\":\"section\",\"featured\":{\"featuredDescription\":\"Honeycomb was built for the AI era. Learn how to futureproof your software for what comes next.\",\"featuredLink\":{\"_type\":\"link\",\"internalLink\":{\"_id\":\"d5963fdd-59ef-4656-9227-e8185cd2a2f3\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"alt\":\"The observability platform for the AI era\",\"asset\":{\"_ref\":\"image-63a3ac161c99c1796fc5a7974c3d092f1cfa33b8-1440x795-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"platform\",\"summary\":null,\"title\":\"Platform\"},\"target\":\"_self\",\"text\":\"See overview\"},\"featuredTitle\":\"Explore the platform\"},\"sectionTitle\":\"Observability Platform\",\"submenues\":[{\"_key\":\"6f71ed942fc9\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"3a7e355c0c21\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"8812e19c-592e-4c61-84c1-2651e0a954d9\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/distributed-tracing\",\"summary\":null,\"title\":\"Distributed Tracing\"},\"target\":\"_self\",\"text\":\"Distributed Tracing\"},{\"_key\":\"57c3e96709fa\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"d0b34557-8feb-4ba5-b2eb-01f432178b13\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/log-analytics\",\"summary\":null,\"title\":\"Log Analytics\"},\"target\":\"_self\",\"text\":\"Log Analytics\"},{\"_key\":\"7c7fa92d4ab4\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"bbf5cba4-e312-413d-a408-3309d4658981\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/metrics\",\"summary\":null,\"title\":\"Metrics\"},\"target\":\"_self\",\"text\":\"Time Series Metrics\"},{\"_key\":\"62cd2b120845\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"075a1a4e-92e4-4f79-8467-c26c60d13c9f\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/frontend-observability\",\"summary\":null,\"title\":\"Frontend Observability\"},\"target\":\"_self\",\"text\":\"Frontend Observability\"},{\"_key\":\"1c28ef24a5cd\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"1823678b-ccc8-4979-8c9f-462d7ec3e2d2\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/telemetry-pipeline\",\"summary\":null,\"title\":\"Telemetry Pipeline\"},\"target\":\"_self\",\"text\":\"Telemetry Pipeline\"},{\"_key\":\"a352c53a8036\",\"_type\":\"link\",\"anchorLink\":\"/platform/private-cloud\",\"enableDownloadFunctionality\":false,\"internalLink\":null,\"target\":\"_self\",\"text\":\"Private Cloud\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"internalLink\":{\"_id\":\"d5963fdd-59ef-4656-9227-e8185cd2a2f3\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"alt\":\"The observability platform for the AI era\",\"asset\":{\"_ref\":\"image-63a3ac161c99c1796fc5a7974c3d092f1cfa33b8-1440x795-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"platform\",\"summary\":null,\"title\":\"Platform\"},\"target\":\"_self\",\"text\":\"Core Observability\"}}},{\"_key\":\"776067ac438a\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"6218cd7e67db\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":{\"_id\":\"17b07749-37e6-434c-81fe-496d29fdac7b\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/agent-timeline\",\"summary\":null,\"title\":\"Agent Timeline\"},\"target\":\"_self\",\"text\":\"Agent Timeline\"},{\"_key\":\"4f3399612de8\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":{\"_id\":\"152efaa5-299f-4c15-818b-7dda6faaf550\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"use-cases/ai-llm-observability\",\"summary\":null,\"title\":\"AI \u0026 LLMs\"},\"target\":\"_self\",\"text\":\"LLM Observability\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":null,\"target\":\"_self\"},\"string\":\"AI Agent Observability\"}},{\"_key\":\"f930b4882a79\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"bafda870cc04\",\"_type\":\"link\",\"anchorLink\":\"/platform/canvas\",\"enableDownloadFunctionality\":false,\"internalLink\":null,\"target\":\"_self\",\"text\":\"Canvas\"},{\"_key\":\"e4c44c9abcc8\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":{\"_id\":\"099fc215-3076-4b38-9cae-69574de9b848\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"technologies/ai-agents\",\"summary\":null,\"title\":\"AI Agents\"},\"target\":\"_self\",\"text\":\"MCP\"},{\"_key\":\"146b1517edde\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":{\"_id\":\"1fd45d6f-7bde-43bb-a3da-1c97fe37fa5f\",\"_type\":\"blogArticle\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"advanced\":{\"highPriority\":false},\"alt\":\"Accelerate Your OpenTelemetry Migrations With Honeycomb's Agent Skills\",\"asset\":{\"_ref\":\"image-cc8a44b656f324560a0aa7ec2d16dc0f72777a5d-3840x2160-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"accelerate-opentelemetry-migrations-honeycomb-agent-skills\",\"summary\":\"With the release of our Agent Skills, we’ve used our domain knowledge to make your agents even more powerful. Instead of having to copy and paste blocks of markdown into Claude Code, we've distilled everything we know about using Honeycomb into a set of core skills, packaged it up, and are making it open source today!\",\"title\":\"Accelerate Your OpenTelemetry Migrations With Honeycomb's Agent Skills\"},\"target\":\"_self\",\"text\":\"MCP Skills\"},{\"_key\":\"89cfc92cfad2\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":{\"_id\":\"733a21e1-1066-4e73-b140-7dc621bdd3bc\",\"_type\":\"blogArticle\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"alt\":\"Introducing Anomaly Detection: Your Early Warning System for Service Health\",\"asset\":{\"_ref\":\"image-39c5ee873d3c0bef49f04aacf5fdd02d8fa69399-1920x1080-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"introducing-anomaly-detection-early-warning-system-service-health\",\"summary\":\"Today, we're excited to announce the release of Anomaly Detection (currently in alpha), Honeycomb's proactive approach to understanding and acting on service health.\",\"title\":\"Introducing Anomaly Detection: Your Early Warning System for Service Health\"},\"target\":\"_self\",\"text\":\"Anomaly Detection\"}],\"submenuTitle\":{\"_type\":\"object\",\"link\":{\"_type\":\"link\",\"internalLink\":{\"_id\":\"58f6403e-b402-449a-90d4-57d9c0122631\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/intelligence\",\"summary\":null,\"title\":\"Honeycomb Intelligence\"},\"target\":\"_self\",\"text\":\"Agentic Intelligence\"}}},{\"_key\":\"670154802905\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"043ac481e220\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"7ce2e31d-1fab-44a2-9588-7fe48fa624fe\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/slos\",\"summary\":null,\"title\":\"SLOs\"},\"target\":\"_self\",\"text\":\"SLOs\"},{\"_key\":\"ad585cbbd043\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"bd42c0ad-4815-4c0d-9eab-329d61d3255b\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/service-map\",\"summary\":null,\"title\":\"Service Map\"},\"target\":\"_self\",\"text\":\"Service Map\"},{\"_key\":\"bafe0c86bbaf\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"63e6273c-ad42-4ca7-84d1-f2add5448f13\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/bubbleup\",\"summary\":null,\"title\":\"BubbleUp\"},\"target\":\"_self\",\"text\":\"BubbleUp\"},{\"_key\":\"e8757ff0a28a\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"e97d4cd4-a2fd-4c54-b847-14cbe27cb025\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/opentelemetry\",\"summary\":null,\"title\":\"OpenTelemetry\"},\"target\":\"_self\",\"text\":\"OpenTelemetry\"},{\"_key\":\"d1f9cc290db7\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"e69a7bb2-8b05-4a53-9d1d-c86f958b719b\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/integrations\",\"summary\":null,\"title\":\"Honeycomb Integrations\"},\"target\":\"_self\",\"text\":\"App Integrations\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"internalLink\":null,\"target\":\"_self\"},\"string\":\"Built-in Features\"}}]},{\"_key\":\"afac2aa97881\",\"_type\":\"section\",\"featured\":{\"featuredDescription\":\"Discover why Honeycomb is the better choice for your engineers, your customers, and your bottom line.\",\"featuredLink\":{\"_type\":\"link\",\"internalLink\":{\"_id\":\"d52bd9a3-ba69-4820-9781-69ceb0477d0b\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"alt\":\"Why Honeycomb? We were built for this.\",\"asset\":{\"_ref\":\"image-aed26058103b0dab7df8f82907350fcbb72a6b60-2880x1590-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"why-honeycomb\",\"summary\":null,\"title\":\"Why Honeycomb?\"},\"target\":\"_self\",\"text\":\"Learn More\"},\"featuredTitle\":\"Why Honeycomb\"},\"sectionTitle\":\"Solutions\",\"submenues\":[{\"_key\":\"a62453edc69e\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"973236213b56\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"e97d4cd4-a2fd-4c54-b847-14cbe27cb025\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/opentelemetry\",\"summary\":null,\"title\":\"OpenTelemetry\"},\"target\":\"_self\",\"text\":\"OpenTelemetry\"},{\"_key\":\"31d4054b0539\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"2de474c6-ecb5-4c29-a908-36e791461ac3\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"technologies/aws\",\"summary\":null,\"title\":\"AWS\"},\"target\":\"_self\",\"text\":\"Amazon Web Services\"},{\"_key\":\"e4533f4323a3\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"d75068e4-4e97-49fc-a85a-c00e5e2494b9\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"technologies/microsoft-azure\",\"summary\":null,\"title\":\"Microsoft Azure\"},\"target\":\"_self\",\"text\":\"Microsoft Azure\"},{\"_key\":\"37dbd8aeb8c4\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"bcff775b-f39f-4caf-a5c5-2923ddcc6636\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"technologies/kubernetes\",\"summary\":null,\"title\":\"Kubernetes\"},\"target\":\"_self\",\"text\":\"Kubernetes\"},{\"_key\":\"6413768f27dc\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"d122bbbc-30d1-4fe8-affc-e85ea473483b\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"technologies/google-cloud\",\"summary\":null,\"title\":\"Google Cloud\"},\"target\":\"_self\",\"text\":\"Google Cloud\"},{\"_key\":\"e3ba65b3e56a\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"099fc215-3076-4b38-9cae-69574de9b848\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"technologies/ai-agents\",\"summary\":null,\"title\":\"AI Agents\"},\"target\":\"_self\",\"text\":\"AI Agents\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"internalLink\":null,\"target\":\"_self\"},\"string\":\"Technologies\"}},{\"_key\":\"83b3f15dfc78\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"12dfc27e1b09\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"152efaa5-299f-4c15-818b-7dda6faaf550\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"use-cases/ai-llm-observability\",\"summary\":null,\"title\":\"AI \u0026 LLMs\"},\"target\":\"_self\",\"text\":\"LLM Observability\"},{\"_key\":\"ag3nt0bs3rv01\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"8f7dab23-a9ba-4218-9ab8-a6afee24d87d\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"use-cases/agent-observability\",\"summary\":null,\"title\":\"Agent Observability\"},\"target\":\"_self\",\"text\":\"Agent Observability\"},{\"_key\":\"cdb5d85b653f\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"de213f02-45ba-46b3-b18f-596eadce44f0\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"use-cases/incident-response\",\"summary\":null,\"title\":\"Incident Response\"},\"target\":\"_self\",\"text\":\"Incident Response\"},{\"_key\":\"05d3787884f2\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"74a75520-566a-4331-a9d7-bf1996e3ce2d\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"use-cases/predictable-costs\",\"summary\":null,\"title\":\"Predictable Costs\"},\"target\":\"_self\",\"text\":\"Predictable Costs\"},{\"_key\":\"d56b2265ff47\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"ffd53dbd-e402-4439-b8f2-ef84b0a12e92\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"use-cases/devops-releases\",\"summary\":null,\"title\":\"DevOps \u0026 Releases\"},\"target\":\"_self\",\"text\":\"DevOps \u0026 Releases\"},{\"_key\":\"687be93e73cd\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"075a1a4e-92e4-4f79-8467-c26c60d13c9f\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"platform/frontend-observability\",\"summary\":null,\"title\":\"Frontend Observability\"},\"target\":\"_self\",\"text\":\"Frontend Development\"},{\"_key\":\"fd9b0d71772c\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"09c23c9b-deb8-45be-a756-c0809bd1e91c\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"use-cases/meet-customer-experience-slas\",\"summary\":null,\"title\":\"Customer SLAs\"},\"target\":\"_self\",\"text\":\"Meet Customer SLAs\"},{\"_key\":\"64e59d6ab8aa\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":{\"_id\":\"d43692a4-4059-437e-bf22-46583f24dcc4\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"use-cases/cloud-migrations\",\"summary\":null,\"title\":\"Cloud Migrations\"},\"target\":\"_self\",\"text\":\"Cloud Migrations\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"internalLink\":null,\"target\":\"_self\"},\"string\":\"Use Cases\"}},{\"_key\":\"3f181a3651da\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"ece052132a81\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"ai-startups-industry\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"industries/ai-startups\",\"summary\":null,\"title\":\"AI Startups\"},\"target\":\"_self\",\"text\":\"AI Startups\"},{\"_key\":\"8e6dab1d6286\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"e2c44aff-8668-4e3a-8120-b62aa3eb40dc\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"industries/financial-services\",\"summary\":null,\"title\":\"Financial Services\"},\"target\":\"_self\",\"text\":\"Financial Services\"},{\"_key\":\"440f49eafb36\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"b2922d61-2d28-4d9e-8368-05b6fb77d703\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"industries/retail\",\"summary\":null,\"title\":\"Retail \u0026 Ecommerce\"},\"target\":\"_self\",\"text\":\"Retail \u0026 Ecommerce\"},{\"_key\":\"9cb0aa2dcebe\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"software-technology-industry\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"industries/software-technology\",\"summary\":null,\"title\":\"Software \u0026 Technology\"},\"target\":\"_self\",\"text\":\"Software \u0026 Technology\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"internalLink\":null,\"target\":\"_self\"},\"string\":\"Industries\"}},{\"_key\":\"fa238aa2fe57\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"91d95a07e165\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"be12278f-29bc-456e-9b70-0293181287f2\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"why-honeycomb/customers\",\"summary\":null,\"title\":\"Customers\"},\"target\":\"_self\",\"text\":\"Customer Stories\"},{\"_key\":\"05a8b16c4187\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"d133a33b-dfa5-44cc-9c55-55ba076367fe\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"alt\":\"Choosing the right observability platform\",\"asset\":{\"_ref\":\"image-eb6da1c61d2c068135476c019107f6266ce8651d-1844x1088-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"why-honeycomb/comparisons\",\"summary\":null,\"title\":\"Comparisons\"},\"target\":\"_self\",\"text\":\"Comparisons\"},{\"_key\":\"eda3f9bc2fd1\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"29443270-097f-4364-88ff-c3cd79ea74f2\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"why-honeycomb/enterprise\",\"summary\":null,\"title\":\"Enterprise\"},\"target\":\"_self\",\"text\":\"For Enterprise\"},{\"_key\":\"711d9271d964\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":{\"_id\":\"7c8667f3-5e0d-4fb3-ace2-86560e355930\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"advanced\":{\"highPriority\":false},\"alt\":\"Honeycomb Services\",\"asset\":{\"_ref\":\"image-7322ef9523352f9ca9bef3e2f37cce914b519c49-2880x1590-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"why-honeycomb/services\",\"summary\":null,\"title\":\"Honeycomb Services\"},\"target\":\"_self\",\"text\":\"Honeycomb Services\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"internalLink\":{\"_id\":\"d52bd9a3-ba69-4820-9781-69ceb0477d0b\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"alt\":\"Why Honeycomb? We were built for this.\",\"asset\":{\"_ref\":\"image-aed26058103b0dab7df8f82907350fcbb72a6b60-2880x1590-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"why-honeycomb\",\"summary\":null,\"title\":\"Why Honeycomb?\"},\"target\":\"_self\",\"text\":\"Why Honeycomb?\"}}}]},{\"_key\":\"f89507b788f8\",\"_type\":\"section\",\"featured\":{\"featuredDescription\":\"Start your journey with the definitive guide to observability. Download our complimentary ebook.\",\"featuredLink\":{\"_type\":\"link\",\"internalLink\":{\"_id\":\"da07f58b-9f3f-4e77-9ebc-140a67c2dc94\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"alt\":\"Observability Engineering O'Reilly Book\",\"asset\":{\"_ref\":\"image-823053e26c243df14b4e682ba2e91d260e06f401-2880x1590-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"observability-engineering-oreilly-book\",\"summary\":null,\"title\":\"Observability Engineering O'Reilly Book\"},\"target\":\"_self\",\"text\":\"Get your copy\"},\"featuredTitle\":\"Observability Engineering\"},\"sectionTitle\":\"Learn \",\"submenues\":[{\"_key\":\"cb2c7ca3a812\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"0579b3261d13\",\"_type\":\"link\",\"externalLink\":\"https://docs.honeycomb.io/\",\"internalLink\":null,\"target\":\"_blank\",\"text\":\"Docs\"},{\"_key\":\"ed11462fee29\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":{\"_id\":\"da07f58b-9f3f-4e77-9ebc-140a67c2dc94\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"alt\":\"Observability Engineering O'Reilly Book\",\"asset\":{\"_ref\":\"image-823053e26c243df14b4e682ba2e91d260e06f401-2880x1590-png\",\"_type\":\"reference\"}}},\"resourceType\":null,\"slug\":\"observability-engineering-oreilly-book\",\"summary\":null,\"title\":\"Observability Engineering O'Reilly Book\"},\"target\":\"_self\",\"text\":\"Observability Engineering\"},{\"_key\":\"acf97f14bc60\",\"_type\":\"link\",\"externalLink\":\"https://docs.honeycomb.io/get-started\",\"internalLink\":null,\"target\":\"_blank\",\"text\":\"Quickstart\"},{\"_key\":\"a54f159c78a0\",\"_type\":\"link\",\"externalLink\":\"https://docs.honeycomb.io/send-data/\",\"internalLink\":null,\"target\":\"_blank\",\"text\":\"Sending data\"},{\"_key\":\"053ed6ac8255\",\"_type\":\"link\",\"externalLink\":\"https://play.honeycomb.io/sandbox/tours\",\"internalLink\":null,\"target\":\"_blank\",\"text\":\"Sandbox\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"internalLink\":null,\"target\":\"_self\"},\"string\":\"Engineers\"}},{\"_key\":\"30b7f786460e\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"104a20760fd8\",\"_type\":\"link\",\"anchorLink\":\"/blog\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Blog\"},{\"_key\":\"6d821088a12e\",\"_type\":\"link\",\"anchorLink\":\"/resources/getting-started\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Getting Started\"},{\"_key\":\"e4d103599e59\",\"_type\":\"link\",\"anchorLink\":\"/resources/guides\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Technical Guides\"},{\"_key\":\"84b6f5e31734\",\"_type\":\"link\",\"anchorLink\":\"/resources/case-studies\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Case Studies\"},{\"_key\":\"a7b9915d2b3b\",\"_type\":\"link\",\"anchorLink\":\"/resources/webinars\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Webinars\"},{\"_key\":\"a3e557190042\",\"_type\":\"link\",\"anchorLink\":\"/resources/whitepapers\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Whitepapers\"},{\"_key\":\"f6862a0a3b48\",\"_type\":\"link\",\"anchorLink\":\"/resources/product-videos\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Product Videos\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"anchorLink\":\"/resources\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Resource Center\"}}},{\"_key\":\"d0f9a93d85af\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"8a2f7421464f\",\"_type\":\"link\",\"anchorLink\":\"/events\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Events\"},{\"_key\":\"eb8b17f096e8\",\"_type\":\"link\",\"internalLink\":{\"_id\":\"cc965279-3211-46ce-8f2c-bf64b4555a52\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"office-hours\",\"summary\":null,\"title\":\"Observability Office Hours\"},\"target\":\"_self\",\"text\":\"Office Hours\"},{\"_key\":\"bb0360e4b1ad\",\"_type\":\"link\",\"externalLink\":\"https://join.slack.com/t/honeycombpollinators/shared_invite/zt-3zl4f8i68-1PhyiwIIqZ0mCXtJjJcSsg\",\"internalLink\":null,\"target\":\"_blank\",\"text\":\"Pollinators Slack\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"internalLink\":null,\"target\":\"_self\"},\"string\":\"Community\"}},{\"_key\":\"93162a1c4ad9\",\"_type\":\"submenu\",\"submenuLinks\":[{\"_key\":\"b410785737ee\",\"_type\":\"link\",\"externalLink\":\"https://academy.honeycomb.io/app/catalog\",\"internalLink\":null,\"target\":\"_blank\",\"text\":\"Course Catalog\"},{\"_key\":\"eba7082cbcd2\",\"_type\":\"link\",\"externalLink\":\"https://academy.honeycomb.io/app/learning_paths\",\"internalLink\":null,\"target\":\"_blank\",\"text\":\"Learning Paths\"}],\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"externalLink\":\"https://academy.honeycomb.io/\",\"internalLink\":null,\"target\":\"_blank\",\"text\":\"Honeycomb Academy\"}}}]},{\"_key\":\"dfc1d152e74d\",\"_type\":\"section\",\"featured\":{\"featuredDescription\":\"Bring observability to every software engineer.\",\"featuredLink\":{\"_type\":\"link\",\"internalLink\":{\"_id\":\"89eca311-0c6a-4570-95bd-b4f659e6aee1\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"about\",\"summary\":null,\"title\":\"About Us\"},\"target\":\"_self\",\"text\":\"About Us\"},\"featuredTitle\":\"Our mission\"},\"sectionTitle\":\"Company\",\"submenues\":[{\"_key\":\"ffc41c9baeda\",\"_type\":\"submenu\",\"submenuDescription\":\"Learn about our company, mission and values.\",\"submenuLinks\":null,\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"internalLink\":{\"_id\":\"89eca311-0c6a-4570-95bd-b4f659e6aee1\",\"_type\":\"page\",\"download\":null,\"mainImage\":null,\"resourceType\":null,\"slug\":\"about\",\"summary\":null,\"title\":\"About Us\"},\"target\":\"_self\",\"text\":\"About Us\"}}},{\"_key\":\"f42e12d30ae5\",\"_type\":\"submenu\",\"submenuDescription\":\"Come for the impact, stay for the culture.\\n\",\"submenuLinks\":null,\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"anchorLink\":\"/careers\",\"internalLink\":null,\"target\":\"_self\",\"text\":\"Careers\"}}},{\"_key\":\"4c3608a4ece0\",\"_type\":\"submenu\",\"submenuDescription\":\"See Honeycomb's latest press releases, media, and more\",\"submenuLinks\":null,\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"internalLink\":{\"_id\":\"bf3a8b58-5814-4183-92f9-76e631372fb2\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"advanced\":{\"highPriority\":false}}},\"resourceType\":null,\"slug\":\"news\",\"summary\":null,\"title\":\"News\"},\"target\":\"_self\",\"text\":\"News\"}}},{\"_key\":\"47b17347e546\",\"_type\":\"submenu\",\"submenuDescription\":\"Learn more about becoming a Honeycomb partner.\",\"submenuLinks\":null,\"submenuTitle\":{\"link\":{\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"internalLink\":{\"_id\":\"de254d71-5d58-4cda-930a-20a976ea4353\",\"_type\":\"page\",\"download\":null,\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"advanced\":{\"highPriority\":false}}},\"resourceType\":null,\"slug\":\"partners\",\"summary\":null,\"title\":\"Become a Honeycomb Partner\"},\"target\":\"_self\",\"text\":\"Partners\"}}}]},{\"_key\":\"b519e18d3109\",\"_type\":\"link\",\"anchorLink\":\"/pricing\",\"featured\":null,\"internalLink\":null,\"submenues\":null,\"target\":\"_self\",\"text\":\"Pricing\"}]}]}],\"$L22\",\"$L23\",\"$L24\",false]\n"])</script><script>self.__next_f.push([1,"26:I[19717,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\",\"/_next/static/chunks/0ku~nn_l8-fol.js\",\"/_next/static/chunks/039u5_5hlemld.js\",\"/_next/static/chunks/189hrhe.fk4af.js\",\"/_next/static/chunks/0oz-qxk30w~qw.js\",\"/_next/static/chunks/0vx_phl8cla2s.js\",\"/_next/static/chunks/0xywjwd04osrr.js\",\"/_next/static/chunks/0t1~yeqvn891r.js\",\"/_next/static/chunks/0ks4m351.jbv2.js\",\"/_next/static/chunks/01gy5f4286e6a.js\",\"/_next/static/chunks/0h3ejemwt0u_c.js\"],\"\"]\n22:[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L5\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[\"$L25\",[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]\n"])</script><script>self.__next_f.push([1,"23:[\"$\",\"footer\",null,{\"className\":\"original-footer font-roboto text-hc-slate mx-auto max-w-[1152px] px-4\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex w-full flex-col items-center gap-[40px] py-[80px] lg:flex-row lg:items-stretch lg:gap-[132px]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"order-2 flex-1 lg:order-1\",\"children\":[[\"$\",\"div\",null,{\"className\":\"hidden flex-1 flex-row justify-between pb-[74px] lg:flex\",\"children\":[[\"$\",\"div\",\"9b31234550a8\",{\"className\":\"flex flex-col gap-3\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-base leading-[150%] font-semibold\",\"children\":\"Observability Platform\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",\"3709d74ab43c\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/pricing\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"3709d74ab43c\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Pricing\",\"$undefined\"]}]}],[\"$\",\"li\",\"c73c8241288b\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"c73c8241288b\",\"_type\":\"link\",\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Platform Overview\",\"$undefined\"]}]}],[\"$\",\"li\",\"7270c5c87c85\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/frontend-observability\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"7270c5c87c85\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Frontend Observability\",\"$undefined\"]}]}],[\"$\",\"li\",\"db25504b1c78\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/log-analytics\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"db25504b1c78\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Log Analytics\",\"$undefined\"]}]}],[\"$\",\"li\",\"ea87e5e00146\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/distributed-tracing\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"ea87e5e00146\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Distributed Tracing\",\"$undefined\"]}]}],[\"$\",\"li\",\"045ae34eb65d\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/metrics\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"045ae34eb65d\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Metrics\",\"$undefined\"]}]}],[\"$\",\"li\",\"3d1692a51e34\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/telemetry-pipeline\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"3d1692a51e34\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Telemetry Pipeline\",\"$undefined\"]}]}],[\"$\",\"li\",\"4196f597a92d\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/private-cloud\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"4196f597a92d\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Private Cloud\",\"$undefined\"]}]}],[\"$\",\"li\",\"b22d2cde4a3d\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/slos\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"b22d2cde4a3d\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"SLOs\",\"$undefined\"]}]}],[\"$\",\"li\",\"0f5a9a5f7aa7\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/service-map\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"0f5a9a5f7aa7\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Service Map\",\"$undefined\"]}]}],[\"$\",\"li\",\"6db0fbacbbc4\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/bubbleup\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"6db0fbacbbc4\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"BubbleUp\",\"$undefined\"]}]}],[\"$\",\"li\",\"d672da05edda\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/opentelemetry\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"d672da05edda\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"OpenTelemetry\",\"$undefined\"]}]}],[\"$\",\"li\",\"80b1b84cfd32\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/platform/integrations\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"80b1b84cfd32\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"App Integrations\",\"$undefined\"]}]}]]}]]}],\"$L27\",\"$L28\",\"$L29\"]}],\"$L2a\"]}],\"$L2b\"]}]}]\n"])</script><script>self.__next_f.push([1,"24:[\"$\",\"footer\",null,{\"className\":\"font-roboto text-hc-slate small-footer container hidden flex-col items-center justify-between gap-10 py-10 lg:flex-row lg:gap-5 xl:gap-0\",\"children\":[[\"$\",\"ul\",null,{\"className\":\"flex flex-1 flex-col justify-between gap-5 text-center lg:flex-2/3 lg:flex-row lg:gap-0 lg:text-left\",\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L26\",null,{\"href\":\"https://honeycomb.io\",\"className\":\"text-hc-cobalt leading-[20px] tracking-[0.02em] underline\",\"children\":\"Honeycomb.io\"}]}],[[\"$\",\"li\",\"4042940388e7\",{\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/terms\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"4042940388e7\",\"_type\":\"link\",\"className\":\"text-hc-cobalt leading-[20px] tracking-[0.02em] text-nowrap underline\",\"children\":[\"Terms of Service\",\"$undefined\"]}]}],[\"$\",\"li\",\"df3d0de1abf9\",{\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/acceptable-use-policy\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"df3d0de1abf9\",\"_type\":\"link\",\"className\":\"text-hc-cobalt leading-[20px] tracking-[0.02em] text-nowrap underline\",\"children\":[\"Acceptable Use Policy\",\"$undefined\"]}]}],[\"$\",\"li\",\"de1418f17f9f\",{\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/privacy\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"de1418f17f9f\",\"_type\":\"link\",\"className\":\"text-hc-cobalt leading-[20px] tracking-[0.02em] text-nowrap underline\",\"children\":[\"Privacy Notice\",\"$undefined\"]}]}]],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L26\",null,{\"href\":\"#\",\"className\":\"cky-banner-element text-hc-cobalt leading-[20px] tracking-[0.02em] text-nowrap underline\",\"children\":\"Your Privacy Choices\"}]}]]}],[\"$\",\"span\",null,{\"className\":\"text-end leading-[20px] tracking-[0.02em] lg:max-w-fit xl:max-w-none xl:flex-1/3\",\"children\":[\"© \",2026,\" \",\"Hound Technology, Inc.\"]}]]}]\n"])</script><script>self.__next_f.push([1,"2c:I[115353,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\",\"/_next/static/chunks/0ku~nn_l8-fol.js\",\"/_next/static/chunks/039u5_5hlemld.js\",\"/_next/static/chunks/189hrhe.fk4af.js\",\"/_next/static/chunks/0oz-qxk30w~qw.js\",\"/_next/static/chunks/0vx_phl8cla2s.js\",\"/_next/static/chunks/0xywjwd04osrr.js\",\"/_next/static/chunks/0t1~yeqvn891r.js\",\"/_next/static/chunks/0ks4m351.jbv2.js\",\"/_next/static/chunks/01gy5f4286e6a.js\"],\"default\"]\n2d:I[995163,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\",\"/_next/static/chunks/0ku~nn_l8-fol.js\",\"/_next/static/chunks/039u5_5hlemld.js\",\"/_next/static/chunks/189hrhe.fk4af.js\",\"/_next/static/chunks/0oz-qxk30w~qw.js\",\"/_next/static/chunks/0vx_phl8cla2s.js\",\"/_next/static/chunks/0xywjwd04osrr.js\",\"/_next/static/chunks/0t1~yeqvn891r.js\",\"/_next/static/chunks/0ks4m351.jbv2.js\",\"/_next/static/chunks/01gy5f4286e6a.js\",\"/_next/static/chunks/0h3ejemwt0u_c.js\"],\"Image\"]\n"])</script><script>self.__next_f.push([1,"27:[\"$\",\"div\",\"a6c0cb4a869f\",{\"className\":\"flex flex-col gap-3\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-base leading-[150%] font-semibold\",\"children\":\"Solutions\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",\"8f77d4ed6602\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/why-honeycomb\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"8f77d4ed6602\",\"_type\":\"link\",\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Why Honeycomb\",\"$undefined\"]}]}],[\"$\",\"li\",\"7fb03607561c\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/why-honeycomb/enterprise\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"7fb03607561c\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Enterprise\",\"$undefined\"]}]}],[\"$\",\"li\",\"d0d6163b27ff\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/technologies/ai-agents\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"d0d6163b27ff\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"AI Agents\",\"$undefined\"]}]}],[\"$\",\"li\",\"2b90f65f46e6\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/technologies/aws\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"2b90f65f46e6\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Amazon Web Services\",\"$undefined\"]}]}],[\"$\",\"li\",\"c3dff0996d2b\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/technologies/microsoft-azure\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"c3dff0996d2b\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Microsoft Azure\",\"$undefined\"]}]}],[\"$\",\"li\",\"d0caa1e4eb66\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/technologies/kubernetes\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"d0caa1e4eb66\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Kubernetes\",\"$undefined\"]}]}],[\"$\",\"li\",\"9ceaefb31039\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/technologies/google-cloud\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"9ceaefb31039\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Google Cloud\",\"$undefined\"]}]}]]}]]}]\n"])</script><script>self.__next_f.push([1,"28:[\"$\",\"div\",\"617021cf9ee2\",{\"className\":\"flex flex-col gap-3\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-base leading-[150%] font-semibold\",\"children\":\"Company\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",\"26df551d2e36\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/about\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"26df551d2e36\",\"_type\":\"link\",\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"About Us\",\"$undefined\"]}]}],[\"$\",\"li\",\"cd5a23b41429\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/about/team\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"cd5a23b41429\",\"_type\":\"link\",\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Team\",\"$undefined\"]}]}],[\"$\",\"li\",\"9e8d0ae5c80c\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/careers\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"9e8d0ae5c80c\",\"_type\":\"link\",\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Careers\",\"$undefined\"]}]}],[\"$\",\"li\",\"21e1fb267d5e\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/news\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"21e1fb267d5e\",\"_type\":\"link\",\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"News\",\"$undefined\"]}]}],[\"$\",\"li\",\"bb5ab8caffa9\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/events\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"bb5ab8caffa9\",\"_type\":\"link\",\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Events\",\"$undefined\"]}]}],[\"$\",\"li\",\"f8a10f88008f\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/why-honeycomb/customers\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"f8a10f88008f\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Customers\",\"$undefined\"]}]}],[\"$\",\"li\",\"bf0deb38bf75\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/partners\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"bf0deb38bf75\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Partners\",\"$undefined\"]}]}],[\"$\",\"li\",\"15f6dabb07e6\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"https://docs.honeycomb.io/security-compliance\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"onClick\":\"$undefined\",\"_key\":\"15f6dabb07e6\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Security\",\"$undefined\"]}]}],[\"$\",\"li\",\"85057eb711e5\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/support\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"85057eb711e5\",\"_type\":\"link\",\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Support\",\"$undefined\"]}]}],[\"$\",\"li\",\"e94d32b2d070\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"https://status.honeycomb.io/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"onClick\":\"$undefined\",\"_key\":\"e94d32b2d070\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Status\",\"$undefined\"]}]}],[\"$\",\"li\",\"0ea03725759b\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"https://changelog.honeycomb.io/\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"0ea03725759b\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Changelog\",\"$undefined\"]}]}]]}]]}]\n"])</script><script>self.__next_f.push([1,"29:[\"$\",\"div\",\"84f2e7d94ff8\",{\"className\":\"flex flex-col gap-3\",\"children\":[[\"$\",\"span\",null,{\"className\":\"text-base leading-[150%] font-semibold\",\"children\":\"Resources\"}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",\"671f18da9b3f\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/blog\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"671f18da9b3f\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Blog\",\"$undefined\"]}]}],[\"$\",\"li\",\"92ccb060bb8f\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"https://docs.honeycomb.io\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"92ccb060bb8f\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Documentation\",\"$undefined\"]}]}],[\"$\",\"li\",\"db87d691804a\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/resources/getting-started\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"db87d691804a\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Getting Started\",\"$undefined\"]}]}],[\"$\",\"li\",\"ba793c2d2aec\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/resources/case-studies\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"ba793c2d2aec\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Case Studies\",\"$undefined\"]}]}],[\"$\",\"li\",\"b6324b7fa3ad\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/why-honeycomb/comparisons\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"b6324b7fa3ad\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Comparisons\",\"$undefined\"]}]}],[\"$\",\"li\",\"8dacd4366e92\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/resources/webinars\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"8dacd4366e92\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"Webinars\",\"$undefined\"]}]}],[\"$\",\"li\",\"2e34173dd022\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/blog/what-is-observability-key-components-best-practices\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"2e34173dd022\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"What Is Observability?\",\"$undefined\"]}]}],[\"$\",\"li\",\"52309d8a3d42\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/resources/getting-started/what-is-ai-observability\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"52309d8a3d42\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"What Is AI Observability?\",\"$undefined\"]}]}],[\"$\",\"li\",\"a1a79caf1ea6\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/resources/getting-started/what-is-llm-observability\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"a1a79caf1ea6\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"What Is LLM Observability?\",\"$undefined\"]}]}],[\"$\",\"li\",\"9cf8c6d40ac4\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/resources/getting-started/application-performance-monitoring-vs-observability\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"9cf8c6d40ac4\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"APM vs Observability\",\"$undefined\"]}]}],[\"$\",\"li\",\"b20174d4157b\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/resources/getting-started/getting-started-with-opentelemetry\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"b20174d4157b\",\"_type\":\"link\",\"enableDownloadFunctionality\":false,\"className\":\"text-sm leading-[20px] tracking-[0.02em]\",\"children\":[\"OpenTelemetry Guide\",\"$undefined\"]}]}]]}]]}]\n"])</script><script>self.__next_f.push([1,"2a:[\"$\",\"div\",null,{\"className\":\"flex flex-col items-baseline gap-2 text-xs leading-[150%] tracking-[0] lg:flex-row lg:gap-3\",\"children\":[[\"$\",\"span\",null,{\"children\":[\"© \",2026,\" \",\"Hound Technology, Inc.\"]}],[\"$\",\"ul\",null,{\"className\":\"flex flex-row gap-3\",\"children\":[[[\"$\",\"li\",\"4042940388e7\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/terms\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"4042940388e7\",\"_type\":\"link\",\"className\":\"text-hc-cobalt text-xs leading-[20px] tracking-[0.02em] underline\",\"children\":[\"Terms of Service\",\"$undefined\"]}]}],[\"$\",\"li\",\"df3d0de1abf9\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/acceptable-use-policy\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"df3d0de1abf9\",\"_type\":\"link\",\"className\":\"text-hc-cobalt text-xs leading-[20px] tracking-[0.02em] underline\",\"children\":[\"Acceptable Use Policy\",\"$undefined\"]}]}],[\"$\",\"li\",\"de1418f17f9f\",{\"className\":\"pb-1\",\"children\":[\"$\",\"$L26\",null,{\"ref\":\"$undefined\",\"href\":\"/privacy\",\"target\":\"_self\",\"rel\":\"$undefined\",\"onClick\":\"$undefined\",\"_key\":\"de1418f17f9f\",\"_type\":\"link\",\"className\":\"text-hc-cobalt text-xs leading-[20px] tracking-[0.02em] underline\",\"children\":[\"Privacy Notice\",\"$undefined\"]}]}]],[\"$\",\"li\",null,{\"className\":\"pb-1\",\"children\":[\"$\",\"a\",null,{\"href\":\"#\",\"className\":\"text-hc-cobalt cky-banner-element text-xs leading-[20px] tracking-[0.02em] underline\",\"children\":\"Your Privacy Choices\"}]}]]}]]}]\n"])</script><script>self.__next_f.push([1,"2b:[\"$\",\"div\",null,{\"className\":\"order-1 flex w-full flex-grow flex-col items-center justify-between self-stretch lg:order-2 lg:max-w-[295px]\",\"children\":[[\"$\",\"div\",null,{\"id\":\"footer-newsletter\",\"className\":\"flex-1 lg:px-4\",\"children\":[[\"$\",\"p\",null,{\"className\":\"font-roboto pb-[15px] text-center text-lg leading-[22px] font-bold tracking-[0.02em]\",\"children\":\"Sign up for updates from Honeycomb\"}],[\"$\",\"$L2c\",null,{}]]}],[\"$\",\"div\",null,{\"className\":\"mt-auto pt-4\",\"children\":[\"$\",\"div\",null,{\"className\":\"mt-4 flex gap-4\",\"children\":[[\"$\",\"a\",\"c9875f6cb729\",{\"href\":\"https://bsky.app/profile/honeycomb.io\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"aria-label\":\"Bluesky\",\"className\":\"transition-opacity hover:opacity-80\",\"children\":[\"$\",\"div\",null,{\"className\":\"border-hc-slate flex size-9 items-center justify-center rounded-full border-[2px]\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/b791a32106224d6a8c17908d9e10a1098ecd033e-600x530.svg\",\"alt\":\"Bluesky\",\"width\":600,\"height\":530,\"className\":\"h-3.5 w-3.5 object-contain\"}]}]}],[\"$\",\"a\",\"b33a8c9306c9\",{\"href\":\"https://www.linkedin.com/company/honeycomb.io\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"aria-label\":\"linkedin\",\"className\":\"transition-opacity hover:opacity-80\",\"children\":[\"$\",\"div\",null,{\"className\":\"border-hc-slate flex size-9 items-center justify-center rounded-full border-[2px]\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/2129b6ff6ad7d0a679bcde4a6d1f3498d2c5c2c3-36x36.svg\",\"alt\":\"Linkedin\",\"width\":36,\"height\":36,\"className\":\"h-3.5 w-3.5 object-contain\"}]}]}],[\"$\",\"a\",\"abce4ce93e4f\",{\"href\":\"https://www.youtube.com/channel/UCty8KGQ3oAP0MQQmLIv7k0Q\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"aria-label\":\"youtube\",\"className\":\"transition-opacity hover:opacity-80\",\"children\":[\"$\",\"div\",null,{\"className\":\"border-hc-slate flex size-9 items-center justify-center rounded-full border-[2px]\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/6949a9f5fcdca86c2505079300c317729031c030-256x256.svg\",\"alt\":\"YouTube\",\"width\":256,\"height\":256,\"className\":\"h-3.5 w-3.5 object-contain\"}]}]}],[\"$\",\"a\",\"1707e02c4aba\",{\"href\":\"https://www.honeycomb.io/rss/blog.xml\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"aria-label\":\"RSS\",\"className\":\"transition-opacity hover:opacity-80\",\"children\":[\"$\",\"div\",null,{\"className\":\"border-hc-slate flex size-9 items-center justify-center rounded-full border-[2px]\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/c0e681e6a520b9ddc9b0beb072859fd9b6e7e20e-800x800.svg\",\"alt\":\"RSS\",\"width\":800,\"height\":800,\"className\":\"h-3.5 w-3.5 object-contain\"}]}]}]]}]}]]}]\n"])</script><script>self.__next_f.push([1,"25:[\"$\",\"main\",null,{\"className\":\"\",\"children\":[false,\"$L2e\"]}]\n"])</script><script>self.__next_f.push([1,"2e:[\"$L2f\",\"$L30\"]\n"])</script><script>self.__next_f.push([1,"31:I[519455,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\",\"/_next/static/chunks/0ku~nn_l8-fol.js\",\"/_next/static/chunks/039u5_5hlemld.js\",\"/_next/static/chunks/189hrhe.fk4af.js\",\"/_next/static/chunks/0oz-qxk30w~qw.js\",\"/_next/static/chunks/0vx_phl8cla2s.js\",\"/_next/static/chunks/0xywjwd04osrr.js\",\"/_next/static/chunks/0t1~yeqvn891r.js\",\"/_next/static/chunks/0ks4m351.jbv2.js\",\"/_next/static/chunks/01gy5f4286e6a.js\",\"/_next/static/chunks/0h3ejemwt0u_c.js\"],\"Button\"]\n"])</script><script>self.__next_f.push([1,"2f:[\"$\",\"section\",\"ced8bdaf0197\",{\"id\":\"ced8bdaf0197\",\"data-anchor\":\"$undefined\",\"className\":\"bg-hc-gray-50\",\"style\":{\"backgroundColor\":\"var(--color-hc-gray-50)\"},\"children\":[\"$\",\"div\",null,{\"className\":\"relative mx-auto max-w-screen-xl overflow-hidden pt-8 md:pt-14 lg:h-[695px] lg:pt-0 xl:h-[795px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"md:flex md:flex-col md:items-center lg:h-full lg:flex-1 lg:flex-row lg:justify-between\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mx-6 mb-8 md:mx-12 md:mb-0 md:max-w-[681px] lg:max-w-[457px] lg:min-w-[457px] lg:pb-16 xl:mr-0 xl:max-w-[603px] xl:min-w-[552px] xl:pb-0\",\"children\":[\"$\",\"div\",null,{\"className\":\"z-20 flex flex-col gap-6 mb-0 w-full items-center justify-start space-y-6 text-center md:gap-6 md:space-y-0 lg:mb-0 lg:h-full lg:w-full lg:items-start lg:justify-center lg:gap-8 lg:text-left\",\"children\":[[\"$\",\"span\",null,{\"className\":\"font-roboto text-hc-slate text-sm font-medium tracking-[0.28px] xl:text-base\",\"children\":\"ERROR 404\"}],[\"$\",\"h1\",null,{\"_type\":\"title\",\"className\":\"font-display font-medium text-pretty max-md:max-w-[282px]\",\"children\":[[\"$\",\"span\",\"217aaeafb8bc\",{\"className\":\"title-hero-heading text-4xl tracking-tighter lg:text-6xl\",\"children\":[\"This page could not\u2028 be found\"]}]]}],[\"$\",\"div\",null,{\"className\":\"prose-sm lg:prose-base prose-p:leading-normal prose-p:my-0 prose-ul:pl-0 prose-li:pl-0 prose-li:my-0 w-full items-center justify-center max-w-[488px]\",\"children\":[[\"$\",\"p\",\"c20010b4e260\",{\"children\":[\"Oops! Our bees flew off to pollinate somewhere else.\"]}]]}],[\"$\",\"div\",null,{\"className\":\"flex gap-6 flex-col w-full md:w-auto md:flex-row\",\"children\":[[\"$\",\"$L31\",\"0\",{\"variant\":\"solid\",\"link\":{\"_type\":\"link\",\"anchorLink\":\"/\",\"enableDownloadFunctionality\":false,\"internalLink\":null,\"target\":\"_self\",\"text\":\"Go Home\"},\"icon\":\"$undefined\",\"modalVideo\":{\"_type\":\"videoPlayer\",\"uploadedVideo\":null,\"videoSource\":\"upload\"},\"modalForm\":null}]]}]]}]}],[\"$\",\"div\",null,{\"className\":\"relative w-full shrink-1 lg:h-full lg:w-auto lg:max-w-[840px]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative hidden h-full items-center justify-end md:flex\",\"children\":[\"$\",\"div\",null,{\"className\":\"animate-fadeIn relative w-full\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/6c706e9e99a2517c10e0addd89dcfcfedb72b1e4-1325x737.png\",\"alt\":\"Error 404\",\"width\":1325,\"height\":737,\"className\":\"h-auto w-auto max-w-full\",\"priority\":true}]}]}],[\"$\",\"div\",null,{\"className\":\"relative flex justify-end md:hidden\",\"children\":[\"$\",\"div\",null,{\"className\":\"animate-fadeIn relative w-full\",\"children\":null}]}]]}]]}]}]}]\n"])</script><script>self.__next_f.push([1,"30:[\"$\",\"aside\",\"1d7d38c74938\",{\"id\":\"1d7d38c74938\",\"className\":\"col-span-full py-12 lg:py-24 container\",\"children\":[[\"$\",\"div\",null,{\"className\":\"z-20 flex flex-col items-center lg:w-1/2 lg:gap-8 mx-auto mb-0 gap-8 text-center lg:mb-0\",\"children\":[null,[\"$\",\"h2\",null,{\"_type\":\"title\",\"className\":\"font-medium text-pretty font-poppins text-center text-4xl lg:text-5xl\",\"children\":[[\"$\",\"span\",\"018ecf8d4fec\",{\"className\":\"title-heading-2 tracking-tightest text-4xl lg:text-5xl\",\"children\":[\"Dive deeper\"]}]]}],[\"$\",\"div\",null,{\"className\":\"prose-sm lg:prose-base prose-p:leading-normal prose-p:my-0 prose-ul:pl-0 prose-li:pl-0 prose-li:my-0 w-full items-center justify-center\",\"children\":[[\"$\",\"p\",\"6b5338c3d254\",{\"children\":[\"Learn more about the power and possibilities of Honeycomb.\"]}]]}],null]}],[\"$\",\"div\",null,{\"className\":\"gap-4 py-10 lg:py-16 xl:flex xl:gap-6\",\"children\":[[\"$\",\"div\",null,{\"className\":\"full flex flex-col gap-4 md:flex-row lg:flex-col xl:w-[831px] xl:gap-6\",\"children\":[[\"$\",\"$L26\",\"179a5d4c-2a78-4d1f-a242-22bb1d9b4d3a\",{\"className\":\"flex-1 lg:h-[245px]\",\"href\":\"/resources/podcasts/ep-93-adaptive-sampling-strategies-with-mike-goldsmith\",\"children\":[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-50 flex h-full flex-col gap-6 rounded-4xl p-2.5 lg:flex-row\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative h-[172px] w-auto md:h-[177px] lg:h-[225px] lg:w-[396px] xl:flex-1\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png\",\"fill\":true,\"alt\":\"O11yCast podcast logo: a dark background with a radiating orange, blue, green, and yellow circular pattern above the white and orange text '011Y_Cast(((' and 'Everything Observability'.\",\"className\":\"aspect-video rounded-4xl object-cover\",\"sizes\":\"(max-width: 768px) 100vw, 396px\"}]}],[\"$\",\"div\",null,{\"className\":\"flex flex-1 flex-col gap-4 max-lg:px-2.5 lg:pt-4\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-4\",\"children\":[[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-sky-500 text-sky-50\",\"children\":\"Podcasts\"}],[[\"$\",\"span\",null,{\"className\":\"h-3.5 border-r border-r-black/30\"}],[\"$\",\"span\",null,{\"className\":\"font-medium font-roboto text-hc-slate/70 text-xs\",\"children\":[null,[\"$\",\"span\",null,{\"children\":\"September 3, 2026\"}]]}]]]}],[\"$\",\"h3\",null,{\"className\":\"font-roboto line-clamp-2 text-base font-medium\",\"children\":\"Ep. #93, Adaptive Sampling Strategies with Mike Goldsmith\"}],[\"$\",\"p\",null,{\"className\":\"font-roboto text-hc-slate/70 line-clamp-3 text-sm font-normal max-lg:hidden\",\"children\":\"On episode 93 of O11ycast, Mike Goldsmith talks to Martin Thwaites and Ken Rimple discuss Honeycomb's contribution to the OpenTelemetry Collector, the Adaptive Tail Sampling Processor. This processor was created based on years of experience building Honeycomb's custom, industry leading tail sampler, Refinery.\\n\\n\"}]]}]]}]}],[\"$\",\"$L26\",\"a435bd91-9998-4e2e-af39-89315db18bd0\",{\"className\":\"flex-1 lg:h-[245px]\",\"href\":\"/resources/leading-with-observability/leading-with-observability-scaling-fin-to-2x-engineering-productivity\",\"children\":[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-50 flex h-full flex-col gap-6 rounded-4xl p-2.5 lg:flex-row\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative h-[172px] w-auto md:h-[177px] lg:h-[225px] lg:w-[396px] xl:flex-1\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/ca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg\",\"fill\":true,\"alt\":\"Title slide for the series \\\"Leading With Observability,\\\" Episode 1: \\\"Scaling Fin to 2x Engineering Productivity,\\\" featuring Darragh Curran (CTO, Fin) and Charity Majors (CTO, Honeycomb).\",\"className\":\"aspect-video rounded-4xl object-cover\",\"sizes\":\"(max-width: 768px) 100vw, 396px\"}]}],\"$L32\"]}]}]]}],\"$L33\"]}],\"$L34\"]}]\n"])</script><script>self.__next_f.push([1,"32:[\"$\",\"div\",null,{\"className\":\"flex flex-1 flex-col gap-4 max-lg:px-2.5 lg:pt-4\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-4\",\"children\":[[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-gray-200 text-hc-slate\",\"children\":\"Leading With Observability\"}],[[\"$\",\"span\",null,{\"className\":\"h-3.5 border-r border-r-black/30\"}],[\"$\",\"span\",null,{\"className\":\"font-medium font-roboto text-hc-slate/70 text-xs\",\"children\":[null,[\"$\",\"span\",null,{\"children\":\"August 27, 2026\"}]]}]]]}],[\"$\",\"h3\",null,{\"className\":\"font-roboto line-clamp-2 text-base font-medium\",\"children\":\"Leading With Observability: Scaling Fin to 2x Engineering Productivity\"}],[\"$\",\"p\",null,{\"className\":\"font-roboto text-hc-slate/70 line-clamp-3 text-sm font-normal max-lg:hidden\",\"children\":\"Darragh Curran, CTO at Fin (formerly Intercom), set out to double engineering productivity—and nearly tripled it instead, using AI-driven code, an AI-powered PR review system, observability as a trust mechanism, and much more hands-on leadership. In the first episode of Leading With Observability, Charity gets into the messy bits behind that transformation, not just the highlight reel.\"}]]}]\n"])</script><script>self.__next_f.push([1,"33:[\"$\",\"div\",null,{\"className\":\"full flex flex-col gap-4 max-xl:mt-4 md:flex-col lg:flex-row xl:flex-1 xl:flex-col xl:gap-6\",\"children\":[[\"$\",\"$L26\",\"aws-honeycomb-workshop-ai-investigators\",{\"href\":\"/resources/webinars/aws-honeycomb-workshop-ai-investigators\",\"className\":\"flex-1 lg:h-[130px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-50 flex h-full gap-4 rounded-3xl p-4 xl:gap-6 xl:rounded-4xl xl:p-2.5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative h-[84px] w-[84px] self-start xl:h-[135px] xl:w-[135px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"aspect-square h-full w-full overflow-hidden rounded-2xl bg-white xl:rounded-3xl\",\"children\":[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"viewBox\":\"0 0 542 542\",\"fill\":\"none\",\"className\":\"h-full w-full\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M0 0h542v542H0z\"}],[\"$\",\"path\",null,{\"fill\":\"#F7D2D7\",\"d\":\"M460.213 394.786c38.019-55.443 16.608-179.375-78.922-214.08-56.823-20.676-114.265-101.902-218.578-66.827a195.6 195.6 0 0 0-8.51 3.138 160.962 160.962 0 0 0-8.235 3.692C38.36 174.244 36.576 301.622 141.37 345.065c89.764 37.168 248.431 152.484 318.843 49.721Z\"}],[\"$\",\"rect\",null,{\"width\":318,\"height\":202,\"x\":107,\"y\":166,\"fill\":\"#fff\",\"rx\":5}],[\"$\",\"rect\",null,{\"width\":318,\"height\":202,\"x\":107,\"y\":166,\"stroke\":\"#E4A3A5\",\"strokeWidth\":10,\"rx\":5}],[\"$\",\"circle\",null,{\"cx\":198,\"cy\":243,\"r\":23,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M155 314a43.001 43.001 0 0 1 73.406-30.406A43.001 43.001 0 0 1 241 314h-86Z\"}],[\"$\",\"g\",null,{\"clipPath\":\"url(#a)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M301 187h47.472v47.472H301z\"}],[\"$\",\"circle\",null,{\"cx\":324.737,\"cy\":202.844,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M311.55 224.618a13.19 13.19 0 0 1 13.187-13.187 13.188 13.188 0 0 1 13.187 13.187H311.55Z\"}]]}],[\"$\",\"g\",null,{\"clipPath\":\"url(#b)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M301 243.264h47.472v47.472H301z\"}],[\"$\",\"circle\",null,{\"cx\":324.737,\"cy\":259.108,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M311.55 280.882a13.188 13.188 0 0 1 26.374 0H311.55Z\"}]]}],[\"$\",\"g\",null,{\"clipPath\":\"url(#c)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M301 299.527h47.472v47.472H301z\"}],[\"$\",\"circle\",null,{\"cx\":324.737,\"cy\":315.372,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M311.55 337.145a13.187 13.187 0 0 1 26.374 0H311.55Z\"}]]}],[\"$\",\"g\",null,{\"clipPath\":\"url(#d)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M357.264 187h47.472v47.472h-47.472z\"}],[\"$\",\"circle\",null,{\"cx\":381.001,\"cy\":202.844,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M367.814 224.618a13.188 13.188 0 0 1 13.187-13.187 13.186 13.186 0 0 1 13.187 13.187h-26.374Z\"}]]}],[\"$\",\"g\",null,{\"clipPath\":\"url(#e)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M357.264 243.264h47.472v47.472h-47.472z\"}],[\"$\",\"circle\",null,{\"cx\":381.001,\"cy\":259.108,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M367.814 280.882a13.187 13.187 0 1 1 26.374 0h-26.374Z\"}]]}],[\"$\",\"g\",null,{\"clipPath\":\"url(#f)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M357.264 299.527h47.472v47.472h-47.472z\"}],[\"$\",\"circle\",null,{\"cx\":381.001,\"cy\":315.372,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M367.814 337.145a13.186 13.186 0 1 1 26.374 0h-26.374Z\"}]]}],[\"$\",\"circle\",null,{\"cx\":412.5,\"cy\":188.5,\"r\":53.5,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M434.842 186.768c1.333.77 1.333 2.695 0 3.464l-32.013 18.483c-1.334.77-3-.192-3-1.732v-36.965c0-1.54 1.666-2.502 3-1.733l32.013 18.483Z\"}],[\"$\",\"path\",null,{\"fill\":\"#D75450\",\"d\":\"M238 373h55v27h-55z\"}],[\"$\",\"rect\",null,{\"width\":168,\"height\":27,\"x\":182,\"y\":400,\"fill\":\"#E4A3A5\",\"rx\":13.5}],[\"$\",\"path\",null,{\"stroke\":\"#D75450\",\"strokeLinecap\":\"round\",\"strokeWidth\":5,\"d\":\"M87.805 426.408c5.565 2.626 19.049 5.505 28.461-3.987 9.412-9.491 8.73-19.101 2.011-24.32-5.871-4.135-17.918-1.146-17.026 10.748 1.116 14.868 23.483 16.543 40.084 5.253\"}],[\"$\",\"defs\",null,{\"children\":[[\"$\",\"clipPath\",null,{\"id\":\"a\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M301 187h47.472v47.472H301z\"}]}],[\"$\",\"clipPath\",null,{\"id\":\"b\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M301 243.264h47.472v47.472H301z\"}]}],[\"$\",\"clipPath\",null,{\"id\":\"c\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M301 299.527h47.472v47.472H301z\"}]}],[\"$\",\"clipPath\",null,{\"id\":\"d\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M357.264 187h47.472v47.472h-47.472z\"}]}],[\"$\",\"clipPath\",null,{\"id\":\"e\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M357.264 243.264h47.472v47.472h-47.472z\"}]}],\"$L35\"]}]]}]}]}],\"$L36\"]}]}],\"$L37\",\"$L38\"]}]\n"])</script><script>self.__next_f.push([1,"34:[\"$\",\"div\",null,{\"className\":\"flex w-full justify-center\",\"children\":[\"$\",\"$L31\",null,{\"variant\":\"solid\",\"link\":{\"_type\":\"link\",\"anchorLink\":\"/resources/\",\"enableDownloadFunctionality\":false,\"internalLink\":null,\"target\":\"_self\",\"text\":\"See All Resources\"},\"modalVideo\":{\"_type\":\"videoPlayer\",\"uploadedVideo\":null,\"videoSource\":\"upload\"},\"modalForm\":null}]}]\n"])</script><script>self.__next_f.push([1,"35:[\"$\",\"clipPath\",null,{\"id\":\"f\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M357.264 299.527h47.472v47.472h-47.472z\"}]}]\n36:[\"$\",\"div\",null,{\"className\":\"flex flex-1 flex-col items-start gap-3 xl:pt-[6px]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-purple-100 text-purple-800\",\"children\":\"Webinars\"}],[\"$\",\"h4\",null,{\"className\":\"font-roboto line-clamp-3 text-sm font-medium xl:text-base\",\"children\":\"How Observability Makes Better AI and Human Investigators\"}]]}]\n"])</script><script>self.__next_f.push([1,"37:[\"$\",\"$L26\",\"6dd1ee72-dedf-4b16-b7f1-a5b6f4651390\",{\"href\":\"/resources/getting-started/what-is-contextual-observability\",\"className\":\"flex-1 lg:h-[130px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-50 flex h-full gap-4 rounded-3xl p-4 xl:gap-6 xl:rounded-4xl xl:p-2.5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative h-[84px] w-[84px] self-start xl:h-[135px] xl:w-[135px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"aspect-square h-full w-full overflow-hidden rounded-2xl bg-white xl:rounded-3xl\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/a01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png\",\"fill\":true,\"alt\":\"What Is Contextual Observability? A Practitioner's Guide for Modern Engineering Teams\",\"className\":\"aspect-square rounded-[14px] object-cover xl:rounded-4xl\",\"sizes\":\"(max-width: 768px) 25vw, 135px\"}]}]}],[\"$\",\"div\",null,{\"className\":\"flex flex-1 flex-col items-start gap-3 xl:pt-[6px]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-green-800 text-hc-green-50\",\"children\":\"Getting Started\"}],[\"$\",\"h4\",null,{\"className\":\"font-roboto line-clamp-3 text-sm font-medium xl:text-base\",\"children\":\"What Is Contextual Observability? A Practitioner's Guide for Modern Engineering Teams\"}]]}]]}]}]\n"])</script><script>self.__next_f.push([1,"38:[\"$\",\"$L26\",\"c81456fb-93ee-4ae9-a4b9-9d3633bb10f2\",{\"href\":\"/resources/conference-talks/socratic-ai-integrating-observability-through-interactive-dialogue-o11ycon-2026\",\"className\":\"flex-1 lg:h-[130px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-50 flex h-full gap-4 rounded-3xl p-4 xl:gap-6 xl:rounded-4xl xl:p-2.5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative h-[84px] w-[84px] self-start xl:h-[135px] xl:w-[135px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"aspect-square h-full w-full overflow-hidden rounded-2xl bg-white xl:rounded-3xl\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png\",\"fill\":true,\"alt\":\"Socratic AI: Integrating Observability Through Interactive Dialogue - O11yCon 2026\",\"className\":\"aspect-square rounded-[14px] object-cover xl:rounded-4xl\",\"sizes\":\"(max-width: 768px) 25vw, 135px\"}]}]}],[\"$\",\"div\",null,{\"className\":\"flex flex-1 flex-col items-start gap-3 xl:pt-[6px]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-gold-300 text-gold-1000\",\"children\":\"Conference Talks\"}],[\"$\",\"h4\",null,{\"className\":\"font-roboto line-clamp-3 text-sm font-medium xl:text-base\",\"children\":\"Socratic AI: Integrating Observability Through Interactive Dialogue - O11yCon 2026\"}]]}]]}]}]\n"])</script><script>self.__next_f.push([1,"3b:I[317118,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\"],\"IconMark\"]\n39:T12fa,"])</script><script>self.__next_f.push([1,"{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#article\",\"isPartOf\":{\"@id\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations\"},\"author\":{\"name\":\"Fred Hebert\",\"@id\":\"https://www.honeycomb.io/#/schema/person/fred-hebert\"},\"headline\":\"Incident Report: Exercises, Cleanups, and Evacuations\",\"datePublished\":\"2026-02-25T14:00:00+00:00\",\"dateModified\":\"2026-03-05T14:11:38+00:00\",\"mainEntityOfPage\":{\"@id\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations\"},\"wordCount\":4695,\"publisher\":{\"@id\":\"https://www.honeycomb.io/#organization\"},\"inLanguage\":\"en-US\",\"image\":{\"@id\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#primaryimage\"},\"thumbnailUrl\":\"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png\",\"keywords\":[\"Incident Response\"],\"articleSection\":[\"Incident Response\"]},{\"@type\":\"WebPage\",\"@id\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations\",\"url\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations\",\"name\":\"Incident Report: Exercises, Cleanups, and Evacuations | Honeycomb\",\"isPartOf\":{\"@id\":\"https://www.honeycomb.io/#website\"},\"inLanguage\":\"en-US\",\"primaryImageOfPage\":{\"@id\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#primaryimage\"},\"image\":{\"@id\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#primaryimage\"},\"thumbnailUrl\":\"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png\",\"datePublished\":\"2026-02-25T14:00:00+00:00\",\"dateModified\":\"2026-03-05T14:11:38+00:00\",\"description\":\"On December 5th, 2025, we suffered a major outage in our EU region, with the last recovery steps for it extending until December 17th, 2025. For multiple hours, all of Honeycomb’s event ingestion endpoints were down. Most of the duration was spent in a degraded mode where only Activity Log data was impacted. A general timeline is available on our status page, but in this report, we’ll look at a broader analysis of what happened.\",\"breadcrumb\":{\"@id\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#breadcrumb\"},\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#primaryimage\",\"url\":\"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png\",\"contentUrl\":\"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png\",\"width\":3840,\"height\":2160,\"caption\":\"Incident Report: Exercises, Cleanups, and Evacuations\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https://www.honeycomb.io/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Incident Report: Exercises, Cleanups, and Evacuations\"}]},{\"@type\":\"WebSite\",\"@id\":\"https://www.honeycomb.io/#website\",\"url\":\"https://www.honeycomb.io/\",\"name\":\"Honeycomb\",\"description\":\"Event Driven Debugging\",\"publisher\":{\"@id\":\"https://www.honeycomb.io/#organization\"},\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https://www.honeycomb.io/search?searchTerm={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":\"Organization\",\"@id\":\"https://www.honeycomb.io/#organization\",\"name\":\"Honeycomb\",\"url\":\"https://www.honeycomb.io/\",\"logo\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https://www.honeycomb.io/#/schema/logo/image/\",\"url\":\"https://www.honeycomb.io/api/assets/honeycomb-logo.svg\",\"contentUrl\":\"https://www.honeycomb.io/api/assets/honeycomb-logo.svg\",\"width\":164,\"height\":48,\"caption\":\"Honeycomb\"},\"image\":{\"@id\":\"https://www.honeycomb.io/#/schema/logo/image/\"},\"sameAs\":[\"https://x.com/honeycombio\"]},{\"@type\":\"Person\",\"@id\":\"https://www.honeycomb.io/#/schema/person/fred-hebert\",\"name\":\"Fred Hebert\",\"url\":\"https://www.honeycomb.io/author/fred-hebert\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https://www.honeycomb.io/#/schema/person/image/fred-hebert\",\"url\":\"https://cdn.sanity.io/images/927dxq0h/production/7b06f417ac780bd2e4f02707bab2ec76cfd56665-180x200.png\",\"contentUrl\":\"https://cdn.sanity.io/images/927dxq0h/production/7b06f417ac780bd2e4f02707bab2ec76cfd56665-180x200.png\",\"caption\":\"Fred Hebert\"}}]}"])</script><script>self.__next_f.push([1,"17:[\"$\",\"main\",null,{\"className\":\"mx-auto w-full px-6 py-12 md:container\",\"children\":[[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"$39\"}}],\"$L3a\"]}]\n1a:null\n"])</script><script>self.__next_f.push([1,"1f:[[\"$\",\"title\",\"0\",{\"children\":\"Incident Report: Exercises, Cleanups, and Evacuations\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"Every year, Honeycomb runs disaster recovery scenarios in multiple environments, including in production. Although each of our instances runs in a single region, on at least three Availability Zones (AZs), we have multiple plans for partial regional failures, and particularly, zonal failures.\"}],[\"$\",\"link\",\"2\",{\"rel\":\"canonical\",\"href\":\"https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations\"}],[\"$\",\"meta\",\"3\",{\"property\":\"og:title\",\"content\":\"Incident Report: Exercises, Cleanups, and Evacuations\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:description\",\"content\":\"Every year, Honeycomb runs disaster recovery scenarios in multiple environments, including in production. Although each of our instances runs in a single region, on at least three Availability Zones (AZs), we have multiple plans for partial regional failures, and particularly, zonal failures.\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:site_name\",\"content\":\"Honeycomb\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:locale\",\"content\":\"en_US\"}],[\"$\",\"meta\",\"7\",{\"property\":\"og:image\",\"content\":\"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png?rect=0,77,3840,2006\u0026w=1200\u0026h=627\u0026fit=crop\"}],[\"$\",\"meta\",\"8\",{\"property\":\"og:image:width\",\"content\":\"1200\"}],[\"$\",\"meta\",\"9\",{\"property\":\"og:image:height\",\"content\":\"627\"}],[\"$\",\"meta\",\"10\",{\"property\":\"og:type\",\"content\":\"article\"}],[\"$\",\"meta\",\"11\",{\"property\":\"article:published_time\",\"content\":\"2026-02-25T14:00:00.000Z\"}],[\"$\",\"meta\",\"12\",{\"property\":\"article:modified_time\",\"content\":\"2026-02-25T14:00:00.000Z\"}],[\"$\",\"meta\",\"13\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"14\",{\"name\":\"twitter:title\",\"content\":\"Incident Report: Exercises, Cleanups, and Evacuations\"}],[\"$\",\"meta\",\"15\",{\"name\":\"twitter:description\",\"content\":\"Every year, Honeycomb runs disaster recovery scenarios in multiple environments, including in production. Although each of our instances runs in a single region, on at least three Availability Zones (AZs), we have multiple plans for partial regional failures, and particularly, zonal failures.\"}],[\"$\",\"meta\",\"16\",{\"name\":\"twitter:image\",\"content\":\"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png?rect=0,77,3840,2006\u0026w=1200\u0026h=627\u0026fit=crop\"}],[\"$\",\"meta\",\"17\",{\"name\":\"twitter:image:width\",\"content\":\"1200\"}],[\"$\",\"meta\",\"18\",{\"name\":\"twitter:image:height\",\"content\":\"627\"}],[\"$\",\"link\",\"19\",{\"rel\":\"icon\",\"href\":\"/favicon.ico?favicon.07o~09ebmt8eg.ico\",\"sizes\":\"48x48\",\"type\":\"image/x-icon\"}],[\"$\",\"link\",\"20\",{\"rel\":\"icon\",\"href\":\"/favicon-16x16.png\",\"sizes\":\"16x16\",\"type\":\"image/png\"}],[\"$\",\"link\",\"21\",{\"rel\":\"icon\",\"href\":\"/favicon-32x32.png\",\"sizes\":\"32x32\",\"type\":\"image/png\"}],[\"$\",\"$L3b\",\"22\",{}]]\n"])</script><script>self.__next_f.push([1,"3a:[\"$\",\"article\",null,{\"style\":{\"contain\":\"layout style\"},\"children\":[[\"$\",\"header\",null,{\"className\":\"mx-auto flex max-w-[1116px] flex-col items-start gap-12 md:gap-13\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex w-full flex-col justify-center gap-9 md:gap-[36px]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-col gap-[32px] md:gap-10\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex w-full\",\"children\":[\"$\",\"nav\",null,{\"ref\":\"$undefined\",\"aria-label\":\"breadcrumb\",\"children\":[\"$\",\"ol\",null,{\"ref\":\"$undefined\",\"className\":\"text-hc-slate flex flex-wrap items-center gap-1.5 text-xs break-words sm:gap-2.5\",\"children\":[[\"$\",\"$1\",\"0\",{\"children\":[[\"$\",\"li\",null,{\"ref\":\"$undefined\",\"className\":\"inline-flex items-center gap-1.5\",\"children\":[\"$\",\"a\",null,{\"ref\":\"$undefined\",\"className\":\"font-roboto font-medium transition-colors hover:underline\",\"href\":\"/blog\",\"children\":\"Blog\"}]}],[\"$\",\"li\",null,{\"role\":\"presentation\",\"aria-hidden\":\"true\",\"className\":\"[\u0026\u003esvg]:h-3.5 [\u0026\u003esvg]:w-3.5\",\"children\":[\"$\",\"svg\",null,{\"ref\":\"$undefined\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":24,\"height\":24,\"viewBox\":\"0 0 24 24\",\"fill\":\"none\",\"stroke\":\"currentColor\",\"strokeWidth\":2,\"strokeLinecap\":\"round\",\"strokeLinejoin\":\"round\",\"className\":\"lucide lucide-chevron-right\",\"children\":[[\"$\",\"path\",\"mthhwq\",{\"d\":\"m9 18 6-6-6-6\"}],\"$undefined\"]}]}]]}],[\"$\",\"$1\",\"1\",{\"children\":[[\"$\",\"li\",null,{\"ref\":\"$undefined\",\"className\":\"inline-flex items-center gap-1.5\",\"children\":[\"$\",\"span\",null,{\"ref\":\"$undefined\",\"role\":\"link\",\"aria-disabled\":\"true\",\"aria-current\":\"page\",\"className\":\"text-foreground font-roboto font-medium\",\"children\":\"Incident Report: Exercises, Cleanups, and Evacuations\"}]}],false]}]]}]}]}],[\"$\",\"div\",null,{\"className\":\"flex flex-col gap-[14px]\",\"children\":[null,[\"$\",\"h1\",null,{\"className\":\"w-full text-center text-4xl font-medium tracking-tighter lg:text-6xl\",\"children\":\"Incident Report: Exercises, Cleanups, and Evacuations\"}]]}]]}],[\"$\",\"div\",null,{\"className\":\"flex w-full justify-center text-center\",\"children\":[\"$\",\"p\",null,{\"className\":\"max-w-[872px]\",\"children\":\"On December 5th, 2025, we suffered a major outage in our EU region, with the last recovery steps for it extending until December 17th, 2025. For multiple hours, all of Honeycomb’s event ingestion endpoints were down. Most of the duration was spent in a degraded mode where only Activity Log data was impacted. A general timeline is available on our status page, but in this report, we’ll look at a broader analysis of what happened.\"}]}]]}],[\"$\",\"div\",null,{\"className\":\"flex w-full flex-col gap-3\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex w-full flex-col items-center justify-between gap-5 md:flex-row\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-row items-center px-5 md:px-0\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-[14px]\",\"children\":[[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/7b06f417ac780bd2e4f02707bab2ec76cfd56665-180x200.png?rect=0,10,180,180\u0026w=80\u0026h=80\",\"width\":80,\"height\":80,\"alt\":\"Fred Hebert\",\"className\":\"bg-hc-gray-50 size-10 rounded-full\"}],[\"$\",\"p\",null,{\"className\":\"text-hc-slate text-base\",\"children\":[[\"$\",\"span\",null,{\"children\":\"By: \"}],[\"$\",\"$L26\",null,{\"href\":\"/author/fred-hebert\",\"className\":\"text-hc-cobalt underline hover:no-underline\",\"children\":\"Fred Hebert\"}]]}]]}],[\"$\",\"span\",null,{\"className\":\"text-hc-slate font-roboto text-base font-medium\",\"children\":[[\"$\",\"span\",null,{\"className\":\"ml-1\",\"children\":\" | \"}],[\"$\",\"span\",null,{\"children\":\"February 25, 2026\"}]]}]]}],[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-100 h-0.5 w-full md:hidden\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-wrap gap-3 px-5 md:px-0\",\"children\":[[\"$\",\"$L26\",\"Incident Response,Incident Response\",{\"href\":\"/blog/category/incident-response\",\"children\":[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-gray-200 text-hc-slate\",\"children\":\"Incident Response\"}]}]]}]]}],[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-100 hidden h-0.5 w-full md:block\"}]]}]]}],null,[\"$\",\"div\",null,{\"className\":\"m-auto w-full max-w-[1116px] justify-between lg:flex lg:gap-[80px] lg:py-10\",\"children\":[\"$L3c\",\"$L3d\"]}],null,\"$L3e\",null]}]\n"])</script><script>self.__next_f.push([1,"3f:I[517028,[\"/_next/static/chunks/14_mcc2-.c1ee.js\",\"/_next/static/chunks/019vi3l~x0zoc.js\",\"/_next/static/chunks/0ku~nn_l8-fol.js\",\"/_next/static/chunks/039u5_5hlemld.js\",\"/_next/static/chunks/189hrhe.fk4af.js\",\"/_next/static/chunks/0oz-qxk30w~qw.js\",\"/_next/static/chunks/0vx_phl8cla2s.js\",\"/_next/static/chunks/0xywjwd04osrr.js\",\"/_next/static/chunks/0t1~yeqvn891r.js\",\"/_next/static/chunks/0ks4m351.jbv2.js\",\"/_next/static/chunks/01gy5f4286e6a.js\",\"/_next/static/chunks/0h3ejemwt0u_c.js\"],\"ToC\"]\n"])</script><script>self.__next_f.push([1,"3c:[\"$\",\"div\",null,{\"className\":\"h-fit lg:sticky lg:top-28 lg:w-[280px]\",\"children\":[[\"$\",\"$L3f\",null,{}],[\"$\",\"div\",null,{\"className\":\"hidden overflow-hidden rounded-[22px] bg-gray-50 lg:block\",\"children\":[[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/f45eaaf1f21497ecb581ef36b365c013d0be24d5-2561x2006.jpg\",\"alt\":\"Investigating Mysterious Kafka Broker I/O When Using Confluent Tiered Storage\",\"width\":280,\"height\":184,\"className\":\"h-[184px] object-cover\",\"priority\":true}],[\"$\",\"div\",null,{\"className\":\"p-[30px] pt-5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mb-3 flex items-center gap-4\",\"children\":[[[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-gray-200 text-hc-slate\",\"children\":\"Blog\"}],[\"$\",\"span\",null,{\"className\":\"h-3.5 border-r border-r-black/30\"}]],[\"$\",\"span\",null,{\"className\":\"font-medium font-roboto text-hc-slate/70 text-xs\",\"children\":[null,[\"$\",\"span\",null,{\"children\":\"June 7, 2024\"}]]}]]}],[\"$\",\"div\",null,{\"className\":\"space-y-4\",\"children\":[[\"$\",\"h4\",null,{\"className\":\"font-roboto text-sm font-medium md:text-base\",\"children\":\"Investigating Mysterious Kafka Broker I/O When Using Confluent Tiered Storage\"}],[\"$\",\"p\",null,{\"children\":\"Earlier this year, we upgraded from Confluent Platform 7.0.10 to 7.6.0. While the upgrade went smoothly, there was one thing that was different from previous upgrades: due to changes in the metadata format for Confluent’s Tiered Storage feature, all of our tiered storage metadata files had to be converted to a newer format.\"}],[\"$\",\"$L31\",null,{\"variant\":\"arrow-link\",\"link\":{\"internalLink\":{\"_type\":\"blogArticle\",\"slug\":\"investigating-kafka-tiered-storage\",\"resourceType\":null,\"_id\":\"post-19742\",\"title\":\"Investigating Mysterious Kafka Broker I/O When Using Confluent Tiered Storage\",\"mainImage\":{\"_type\":\"assetImage\",\"image\":{\"_type\":\"image\",\"alt\":\"Investigating Mysterious Kafka Broker I/O When Using Confluent Tiered Storage\",\"asset\":{\"_ref\":\"image-f45eaaf1f21497ecb581ef36b365c013d0be24d5-2561x2006-jpg\",\"_type\":\"reference\"}}},\"summary\":\"Earlier this year, we upgraded from Confluent Platform 7.0.10 to 7.6.0. While the upgrade went smoothly, there was one thing that was different from previous upgrades: due to changes in the metadata format for Confluent’s Tiered Storage feature, all of our tiered storage metadata files had to be converted to a newer format.\",\"download\":null},\"_type\":\"link\"},\"text\":\"Read More\",\"icon\":\"$undefined\",\"modalVideo\":{\"_type\":\"videoPlayer\",\"uploadedVideo\":null,\"videoSource\":\"upload\"},\"modalForm\":null,\"className\":\"[\u0026_span]:text-hc-cobalt [\u0026_svg]:text-hc-cobalt\"}]]}]]}]]}]]}]\n"])</script><script>self.__next_f.push([1,"3d:[\"$\",\"div\",null,{\"className\":\"min-w-0 flex-1\",\"style\":{\"contentVisibility\":\"auto\",\"containIntrinsicSize\":\"auto 1500px\"},\"children\":[[\"$\",\"$L2d\",null,{\"priority\":\"$undefined\",\"fetchPriority\":\"low\",\"loading\":\"lazy\",\"sizes\":\"100vw\",\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/1afacd65863824d6f461263d5ae8789cd9f718f4-3840x2160.png\",\"alt\":\"Incident Report: Exercises, Cleanups, and Evacuations\",\"width\":3840,\"height\":2160,\"className\":\"my-5 block h-auto w-full max-w-full rounded-4xl object-center object-contain\"}],[[\"$\",\"p\",\"ce6e7b614bd0\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Every year, Honeycomb runs disaster recovery scenarios in multiple environments, including in production. Although each of our instances runs in a single region, on at least three Availability Zones (AZs), we have multiple plans for partial regional failures, and particularly, zonal failures. One of these tests was run on December 5th, and after its successful completion came its cleanup steps.\"]}],[\"$\",\"p\",\"984a1c9fab8e\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Because tolerating zonal failures creates and shifts workloads in new zones that are inherently imbalanced once the failure is over, some work is required to gradually shift workloads back in proper balance. During one of these steps, while following our runbook, we ended up purposefully destroying Kafka brokers. In all previous runs in pre-production environments, this step went fine. In production, however, it caused multiple topic partitions to go fully leaderless, which caused major problems at multiple levels:\"]}],[\"$\",\"ul\",\"97d4e0ea5285-parent\",{\"data-testid\":\"article-list-bullet\",\"className\":\"text-hc-slate not-prose font-roboto mt-6 list-disc pl-6 text-sm leading-normal font-normal tracking-normal lg:text-base\",\"children\":[[\"$\",\"li\",\"97d4e0ea5285\",{\"className\":\"mb-2\",\"children\":[\"The topics we use to transmit all ingested events had multiple partitions that were damaged, some of which irreversibly so;\"]}],[\"$\",\"li\",\"8762c6442038\",{\"className\":\"mb-2\",\"children\":[\"The topics used by some of our consumer groups to track where they were at were partially damaged;\"]}],[\"$\",\"li\",\"ae7f7bf69b93\",{\"className\":\"mb-2\",\"children\":[\"Metadata topics used by Kafka and the Confluent addons we use (such as those used to manage tiered storage) were also partially damaged, with some partitions fully broken.\"]}]]}],[\"$\",\"p\",\"477bc7412dfb\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The first element in the list was our top priority: each Honeycomb team and environment is assigned multiple partitions, and while we can sustain some partitions becoming unavailable, if all partitions assigned to a team are unavailable, then we fail to ingest any data for them. But worse than that is the damage these partitions had received.\"]}],[\"$\",\"p\",\"5be8f6b7497b\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"To be brief, Kafka maintains an internal set of offsets that are available to consumers for them to reliably consume data. Retriever, our query engine, takes these offsets and uses them in multiple places internally to track consistency of which events it has processed. If all Kafka brokers for a topic partition are lost, the ensuing leader election needs to be “dirty,” which can partially or completely roll back the offsets. This breaks all Retriever ingestion in the foreseeable future, possibly requiring to delete the environment and start again from scratch.\"]}],\"$L40\",\"$L41\",\"$L42\",\"$L43\",\"$L44\",\"$L45\",\"$L46\",\"$L47\",\"$L48\",\"$L49\",\"$L4a\",\"$L4b\",\"$L4c\",\"$L4d\",\"$L4e\",\"$L4f\",\"$L50\",\"$L51\",\"$L52\",\"$L53\",\"$L54\",\"$L55\",\"$L56\",\"$L57\",\"$L58\",\"$L59\",\"$L5a\",\"$L5b\",\"$L5c\",\"$L5d\",\"$L5e\",\"$L5f\",\"$L60\",\"$L61\",\"$L62\",\"$L63\",\"$L64\",\"$L65\",\"$L66\",\"$L67\",\"$L68\",\"$L69\",\"$L6a\",\"$L6b\",\"$L6c\",\"$L6d\",\"$L6e\",\"$L6f\",\"$L70\",\"$L71\",\"$L72\",\"$L73\",\"$L74\",\"$L75\",\"$L76\",\"$L77\",\"$L78\",\"$L79\",\"$L7a\",\"$L7b\",\"$L7c\",\"$L7d\",\"$L7e\",\"$L7f\",\"$L80\",\"$L81\",\"$L82\",\"$L83\",\"$L84\",\"$L85\",\"$L86\",\"$L87\",\"$L88\",\"$L89\",\"$L8a\",\"$L8b\",\"$L8c\",\"$L8d\",\"$L8e\",\"$L8f\",\"$L90\",\"$L91\",\"$L92\",\"$L93\",\"$L94\",\"$L95\",\"$L96\",\"$L97\",\"$L98\",\"$L99\",\"$L9a\",\"$L9b\",\"$L9c\",\"$L9d\",\"$L9e\",\"$L9f\",\"$La0\",\"$La1\",\"$La2\",\"$La3\",\"$La4\",\"$La5\",\"$La6\",\"$La7\",\"$La8\",\"$La9\",\"$Laa\",\"$Lab\"]]}]\n"])</script><script>self.__next_f.push([1,"40:[\"$\",\"p\",\"d1750918d0d7\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The early response then was split in two: find the teams and environments whose partition assignment needed to be moved (and impact communicated), and see if we could salvage the impacted partitions without offset/data loss.\"]}]\n41:[\"$\",\"p\",\"63daf9d4f326\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Roughly one hour into the incident, the impacted customers were found. The Kafka recovery was a bit trickier: we couldn’t do \",[\"$\",\"em\",\"062de7508e04\",{\"children\":[\"any\"]}],\" random leader election and had to salvage as much data as possible from the offsets that may still exist.\"]}]\n42:[\"$\",\"p\",\"d567e5774f68\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Through the alerting noise, we noticed that our ingest service, Shepherd, had entered a crashloop spiral more than an hour earlier due to increased memory buffering for events that were to be written on the broken partitions.\"]}]\n43:[\"$\",\"p\",\"97fca8ec300a\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Various options to repair ingest for apps on broken partitions were considered, such as adding new ones to all datasets, booting Retrievers in the wrong partitions to create room, and so on. In the end, the impacted partitions were marked read-only on the producer side, which fixed ingest’s crashlooping. Reassignments were then run to bring everything back to normal for customers.\"]}]\n44:[\"$\",\"p\",\"fb8d02a3ba48\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Later in the night, a list of leaderless Retriever partitions (all read-only) with their offsets was established. Roughly a third of the partitions were impacted, but a single one had complete data loss, roughly 2.5% of our cluster.\"]}]\n45:[\"$\",\"p\",\"cc095ecf3ae7\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"As the system was then relatively stable but far from solid, a plan for the night and checking back on Saturday was put in place. The Kafka autobalancer was turned off, and a manual reassignment plan to bring back some balance was put in place, left to run overnight.\"]}]\n46:[\"$\",\"p\",\"9653d04f3635\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Completing reassignments to fresh hosts, for tiered storage specifically, can sometimes take multiple hours of idle-looking time to scan S3 buckets for historical data before any transfer would show up at the network level. We found it concerning to see disk usage going up, but decided to wait.\"]}]\n47:[\"$\",\"p\",\"abef95dfebbb\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Around midnight PT, one partition still showed no sign of recovery: the disk was filling too fast on the brokers for comfort, and so local retention was reduced on Kafka, aiming to use the S3 tiered storage for any longer retention need. A status update was posted, and the group disbanded for the night.\"]}]\n48:[\"$\",\"h2\",\"6af547c0647b\",{\"id\":\"weekend-response\",\"className\":\"text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24\",\"children\":[\"Weekend response\"]}]\n49:[\"$\",\"p\",\"357ee676d9c1\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Early on Saturday morning, responders noticed that the Kafka balancing jobs did not progress, and disk "])</script><script>self.__next_f.push([1,"usage hadn’t improved. Estimates showed we were losing 5% of disk capacity per hour, with only 30% free. To avoid running out of disk on brokers and potentially suffering major data loss that propagates to Retriever, plans were put in place to shut down all ingest at the ALB layer if we reached 95% usage. This left us five hours to figure out a way to free space.\"]}]\n4a:[\"$\",\"p\",\"afc7512cd6c3\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"At that point, the main reassignments were cancelled, and there was an attempt to move smaller partitions to free up space. Responders found out that two topics had broken topic partitions that could be relevant: \",[\"$\",\"code\",\"4eb89fbb84f7\",{\"className\":\"font-mono\",\"children\":[\"_confluent-tier-state\"]}],\" and \",[\"$\",\"code\",\"7ae2e58ebc3c\",{\"className\":\"font-mono\",\"children\":[\"__consumer_offsets\"]}],\". The theory was that since we were trying to move topics and their data to tiered storage, having these broken could prevent proper offloading and uploading of data.\"]}]\n4b:[\"$\",\"p\",\"7bb17bfc8117\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The moment these were repaired, triggers about our SLO processing fired, complaining of ~18h of late data. Our SLO processing service had been unable to see new data had come in based on how it implemented its consumption, and repairing the metadata topics unblocked it. It started catching up.\"]}]\n4c:[\"$\",\"p\",\"d1ed819cda62\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"After multiple efforts, at 7 a.m. PT, with 5.07% of disk storage free, ingest was turned off. Disk usage stabilized, but did not empty—which would have been our expectation if we were uploading older data to S3 and not replacing it with new ingest.\"]}]\n4d:[\"$\",\"p\",\"a85aa2c2f71a\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Honeycomb engineers decided to bring up new Kafka brokers, with new Retrievers, and create brand new partitions to send ingest to.\"]}]\n4e:[\"$\",\"p\",\"b9b0f164921b\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"As this was being rolled out, an engineer came in and suggested we turn off tiered storage altogether to unstick the cleanup process and allow freeing disk space. This is something that had worked in 2021 during an old outage. We turned off tiered storage and disk space was instantly restored.\"]}]\n4f:[\"$\",\"p\",\"15bd6063991e\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"We reopened ingest, and customer impact ended then. Later in the day, as part of cleanup, the team stopped work to manually fix Retriever topic partitions that were fully damaged and truncated them, leaving them in read-only mode. This created a gap where older data was unavailable for querying as Retriever struggled with the offset reset, but it wasn’t lost and we would remediate this problem in the following days.\"]}]\n50:[\"$\",\"h2\",\"9f655e1ea8a5\",{\"id\":\"second-week-response\",\"className\":\"text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24\",\"children\":[\"Second week response\"]}]\n51:[\"$\",\"p\",\"ecd851dc185d\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Since Kafka was still in a rough but functional state, most of Monday’s response focused on two elements:\"]}]\n52:[\"$\",\"ol\",\"b6454a4a719c-parent\",{\"data-testid\":\"article-list-number\",\"className\":\"text-hc-slate not-prose font-roboto mt-6 list-decimal pl-6 text-sm leading-normal font-normal tracking-normal lg:text-base\",\"children\":[[\"$\",\"li\""])</script><script>self.__next_f.push([1,",\"b6454a4a719c\",{\"className\":\"mb-2\",\"children\":[\"Extending the retention of the Honeycomb MySQL readonly replica’s binlog, since Activity Log could no longer publish events to Kafka.\"]}],[\"$\",\"li\",\"1bb64d031be9\",{\"className\":\"mb-2\",\"children\":[\"Salvaging the damaged Kafka topic partition for Retriever.\"]}]]}]\n53:[\"$\",\"p\",\"986f5fa41f2e\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The broken partition was fully offline. Being read-only, no traffic was being lost, but the historical data was not available for querying, although queries succeeded overall, just with partial results.\"]}]\n54:[\"$\",\"p\",\"6fabe9672b72\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Regardless, a series of small tweaks and fixes were put in place to allow the Retriever nodes to idle live and let old data be queried despite “broken” offsets.\"]}]\n55:[\"$\",\"p\",\"b395112e947c\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"On Tuesday, work was done to try and bring back tiered storage; this first required deleting all the old S3 data, which took multiple hours. We did not manage to turn tiered storage back on.\"]}]\n56:[\"$\",\"p\",\"3269002dc36e\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"In parallel, work was done to unfreeze builds that hadn’t moved since Friday. Activity Log repairs got delayed due to more urgent remediation efforts on Generally Available Honeycomb features, though data retention would be available until Friday. We attempted to repair their partitions on Wednesday. After failing to do dirty leader elections in Kafka, we decided to delete the topics involved and recreate them.\"]}]\n57:[\"$\",\"p\",\"b0baeb940f89\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"This is when things got \",[\"$\",\"em\",\"8f154a7dc302\",{\"children\":[\"weird\"]}],\". The delete operation timed out, and suddenly, none of our Kafka brokers could do anything administrative other than list topics. Describing, reassigning, or modifying them would just time out. The cluster was in a far worse state than expected, and responders on the call became uncomfortable.\"]}]\n58:[\"$\",\"p\",\"fbaea46be1cd\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"To move past the feeling, we decided to split approaches: one lane to try and salvage the cluster with non-risky operations and ZooKeeper surgery, and one lane to plan a full evacuation of the cluster to a new one.\"]}]\n59:[\"$\",\"p\",\"228599c8ae29\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Managers and directors were called in to help align the rest of the organization on these efforts. Wednesday afternoon was spent preparing the evacuation efforts (described below) with the support of directors and engineering from all teams that would interact with Kafka either for operations, or through their product area.\"]}]\n5a:[\"$\",\"p\",\"6143efec0969\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Thursday involved the elaboration of multiple evacuation plans, and a division of work while Kafka SMEs kept trying to see what could safely be done about the cluster. Later in the day, even changes such as rebooting the Kafka controller host to shift its responsibility to another broker were seen as high-reward but high-risk, and because evacuation efforts could benefit from prolonged steady state, we tabled these efforts.\"]}]\n5b:[\"$\",\"p\",\"1dc866f68771\""])</script><script>self.__next_f.push([1,",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"On Friday, we realized we wouldn’t be ready in time for Activity Log data to be salvaged in the MySQL binlog, which had a maximal storage time of seven days. Since the feature is still in beta and rushing to save more data through alternative replication means would drag people away from the Kafka evacuation, we chose to try a hail mary approach suggested by AWS support where we’d set up a new replica that was one week old from a snapshot, freeze it with manual replication, and hope that this would trigger AWS’ feature where stalled replication extends binlog retention to up to 30 days.\"]}]\n5c:[\"$\",\"h2\",\"92c55010b9d6\",{\"id\":\"evacuation\",\"className\":\"text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24\",\"children\":[\"Evacuation\"]}]\n5d:[\"$\",\"p\",\"579d0736e4e0\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"An evacuation would essentially ask that we consider the current Kafka cluster as no longer worth salvaging. A new Kafka cluster would be needed, with brand new topics and partitions, all empty with reset offsets. This, in turn, implies modifying or ensuring all of our software components can properly shift from the old to the new cluster without data loss, which would require deep code changes. For example, Retriever needed to be edited so that it was allowed to reset its offset tracking, something that would require changes in some of its internals and a lot of consideration for complicated scenarios.\"]}]\n5e:[\"$\",\"p\",\"2f2a2e9eafa7\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"This sort of effort would be expected to take weeks if not months in normal time, but the incoming holidays break and an already tired team made the deadline short. It was also unclear if the cluster, as it stood in its unmanageable state, would be up for long enough to complete the migration.\"]}]\n5f:[\"$\",\"p\",\"b2a54ea260c8\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"We therefore planned for five options presented summarily here, depending on how much of the preconditions we’d meet:\"]}]\n"])</script><script>self.__next_f.push([1,"60:[\"$\",\"ol\",\"38c12eb9cfd7-parent\",{\"data-testid\":\"article-list-number\",\"className\":\"text-hc-slate not-prose font-roboto mt-6 list-decimal pl-6 text-sm leading-normal font-normal tracking-normal lg:text-base\",\"children\":[[\"$\",\"li\",\"38c12eb9cfd7\",{\"className\":\"mb-2\",\"children\":[\"The existing Kafka cluster dies at the current state of the world. We have a full outage and must recover from scratch. At least four to eight hours of hard downtime expected, higher ongoing costs.\"]}],[\"$\",\"li\",\"e42252ba60b7\",{\"className\":\"mb-2\",\"children\":[\"The existing Kafka cluster dies but we have managed to make Retriever able to handle offset resets; we can recover from scratch without leaving old partitions behind.\"]}],[\"$\",\"li\",\"2dbbf39d20ae\",{\"className\":\"mb-2\",\"children\":[\"The existing Kafka cluster dies while we were ready to migrate to a new cluster. We can reset it, but ingest would be down for one to two hours, querying estimated down for up to three to eight hours.\"]}],[\"$\",\"li\",\"9ea9d3e6346e\",{\"className\":\"mb-2\",\"children\":[\"We’re ready to migrate most things except some minor elements we haven’t had the time to prepare. Zero downtime for ingest, one hour downtime for primary querying and alerting, but may take anywhere between two to eight hours of work—and we were unsure what the impact would be on less critical services.\"]}],[\"$\",\"li\",\"bc241b4dc2c8\",{\"className\":\"mb-2\",\"children\":[\"Full migration as we’re ready. Similar to the previous option, but more testing has happened, we pick some slightly safer implementation details across all applications, and reduce uncertainty.\"]}]]}]\n"])</script><script>self.__next_f.push([1,"61:[\"$\",\"p\",\"a0f6d2fed0ee\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Roughly midday on Friday, the Storage team managed to make the offset reset mechanism work for Retriever. This made the first scenario no longer be a concern.\"]}]\n62:[\"$\",\"p\",\"4c291843253c\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Our platform team had made good progress on the feature flags to switch readers and writers, and also made it possible to boot a second Kafka cluster without interfering with the other ones nor any other environments.\"]}]\n63:[\"$\",\"p\",\"1c9875267400\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"As we progressed through the work on Friday and firefought on some parallel events in the US, we thought we could ambitiously hit the 4th or 5th scenario on Monday afternoon, with a full dress rehearsal in one of our EU pre-prod clusters.\"]}]\n64:[\"$\",\"p\",\"2b043b1e5580\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The dress rehearsal went well with people from more than half a dozen teams on the call. There was some friction around code changes and builds that needed ironing out, but the plan mostly worked fine and was completed within four hours.\"]}]\n65:[\"$\",\"p\",\"91c93fb5277c\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Tuesday required some work to fix issues where domain names for a new Kafka clusters’ NLB wouldn’t fit AWS’ limits, but we managed to fix everything in time. The migration ran in two hours with no significant problem and up to one hour of delayed signal at most.\"]}]\n66:[\"$\",\"p\",\"f86c85498f98\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"We cleaned up the older clusters, and caught up with bits of Activity Log in the EU region. As it turns out, the replica trick didn’t work. To minimize lost data, we booted an instance from Tuesday December 9th, and replicated from there. This kept data loss on the Activity Log from December 5th at ~6 p.m. ET to December 9th at ~6 p.m. ET, and the rest was saved.\"]}]\n67:[\"$\",\"h2\",\"b70d07822dff\",{\"id\":\"analysis\",\"className\":\"text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24\",\"children\":[\"Analysis\"]}]\n68:[\"$\",\"p\",\"358369ade38f\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"This incident was draining,\",[\"$\",\"strong\",\"53313a3eb4d6\",{\"children\":[\" described as “the worst incident of my career”\"]}],\" by many. However, a great part of working for Honeycomb is how caring the organization is toward our incident response team. Frankly, it's at a level many of us hadn’t seen at prior companies.\"]}]\n69:[\"$\",\"p\",\"d484d2cf65cb\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Some engineers mentioned wanting to keep response sustainable, while others felt that it should have required faster escalation.\"]}]\n6a:[\"$\",\"p\",\"3e237db0a2d2\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Coordination proved challenging. Before this happened, multiple senior engineers mentioned feeling like they automatically grabbed the lead when they entered the room, which they mentioned being something they need to think more about.\"]}]\n6b:[\"$\",\"p\",\"d5e325c73464\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-m"])</script><script>self.__next_f.push([1,"t-24\",\"children\":[\"Communication with the public was challenging. We aren’t necessarily used to running such long events, and decided to be explicit about potential losses and pending recovery of Activity Log. While the situation was internally dire, the Activity Log is actually a beta feature; customers let us know that we were giving too much information about minor features.\"]}]\n6c:[\"$\",\"h3\",\"baeec3f49da3\",{\"className\":\"font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24\",\"children\":[\"Documentation fragmentation issues\"]}]\n6d:[\"$\",\"p\",\"0ffcfd090a22\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Many of the elements in play for this outage were already known. Similar outages had been seen in recent years, with high level plans for workarounds, RFCs and migrations that would ease up on these conditions, and processes to run similar exercises safely. Ultimately, these documents’ existence was either unknown (outside of the people who were involved in writing and reviewing them), or inaccessible (blocked by ineffective search on some document systems). Interviews with members of staff revealed a frustration with our documentation situation, spread across multiple areas.\"]}]\n6e:[\"$\",\"p\",\"c72709346c06\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"We see a need to better consolidate our information as we keep growing.\"]}]\n6f:[\"$\",\"h3\",\"21547cf92bb6\",{\"className\":\"font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24\",\"children\":[\"Bumpable roadmap effects\"]}]\n70:[\"$\",\"p\",\"284b22a6fbb8\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"When you are given multiple competing tasks of high importance, the one with a deadline further out or a higher level of uncertainty is easier to defer than the one with a more urgent deadline or well-understood scope. This is often represented as the \",[\"$\",\"$L26\",\"d90fd5c52937\",{\"href\":\"https://en.wikipedia.org/wiki/Time_management#Eisenhower_method\",\"className\":\"not-prose text-hc-cobalt mt-6 font-normal underline hover:no-underline\",\"children\":[\"Eisenhower matrix\"]}],\":\"]}]\n71:[\"$\",\"$L2d\",\"9ca96e754f9a\",{\"priority\":\"$undefined\",\"fetchPriority\":\"low\",\"loading\":\"lazy\",\"sizes\":\"100vw\",\"data-testid\":\"article-image\",\"className\":\"not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full\",\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/a17a14ed08ff4fcfcd69fe58befadecb128fe435-1280x1280.png\",\"alt\":\"Eisenhower Matrix\",\"width\":1280,\"height\":1280}]\n72:[\"$\",\"p\",\"053fa4d42a31\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[[\"$\",\"$L26\",\"22bb2cdacbdb\",{\"href\":\"https://commons.wikimedia.org/w/index.php?curid=141867635\",\"className\":\"not-prose text-hc-cobalt mt-6 font-normal underline hover:no-underline\",\"children\":[\"By Cmglee - Own work, CC BY-SA 4.0\"]}]]}]\n73:[\"$\",\"p\",\"4b68d1b767a2\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"When caught between a clear urgent project that has well-understood customer benefits, it becomes difficult to say no to it in favor of an internally controlled project whose benefits are going to be less obvious or more contextual.\"]}]\n74:[\"$\",\"p\",\"0857e4bfdfc4\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"This gets even muddier when you have to prioritize multiple projects from the same quadrant. It’s at these levels where we can see teams with agency in their own schedules adapt to pressures by deferring o"])</script><script>self.__next_f.push([1,"r prioritizing some projects without going back up to leadership to arbiter common issues of this type.\"]}]\n75:[\"$\",\"p\",\"0d30ab6790c4\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"We could think of the bumpable mechanism as an extra dimension over that matrix: how much control and flexibility there is across competing projects, which makes it possible to reframe or reorder their respective urgency and importance.\"]}]\n76:[\"$\",\"p\",\"d7d76579d1a1\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"In general, being flexible with deadlines is a good thing. It lets us reprioritize as new information becomes available and as we find we need to adjust our plans. However, it appears to be challenging for teams to avoid repeatedly deferring more flexible pieces of work that are locally controlled or have wider margins. This is a convenient and rational way to locally manage tradeoffs.\"]}]\n77:[\"$\",\"p\",\"4d96a9fcca7b\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"We cannot counter all bumping—it plays a useful role to our ability to adapt and adjust—but may want to seek ways of knowing when bumping can become a reflex more than a strategic tool.\"]}]\n78:[\"$\",\"h3\",\"7bad16c65a64\",{\"className\":\"font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24\",\"children\":[\"Sociotechnical misalignment\"]}]\n79:[\"$\",\"p\",\"b8465923cbca\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"A few years ago, Honeycomb was small enough that critical knowledge only existed within single engineers’ minds. We have since then taken measures to ensure all components have owners and that critical knowledge has multiple people behind it. While this helped, such measures need frequent adjusting.\"]}]\n7a:[\"$\",\"p\",\"02d16b2fba25\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Something interesting that came up in the incident is that many key interventions came from people who no longer own the components they helped with. People were able to help because they were here back when teams’ ownership was less well-defined, despite having since moved to different roles. That these people were all able to offer meaningful support is a \",[\"$\",\"em\",\"d5f8c26b0f87\",{\"children\":[\"strength\"]}],\" of this organization. It also shows, however, that there is some drift or gap between the socio- and the -technical part of the system.\"]}]\n7b:[\"$\",\"p\",\"d0d8d8e823a3\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"For example, in creating one team to manage one part of the system and assigning ownership of the other elements to another team, any existing coupling between these components does not vanish. So while both teams can grow a stronger expertise on their respective piece of software, the details get lost:\"]}]\n7c:[\"$\",\"$L2d\",\"b2d309910372\",{\"priority\":\"$undefined\",\"fetchPriority\":\"low\",\"loading\":\"lazy\",\"sizes\":\"100vw\",\"data-testid\":\"article-image\",\"className\":\"not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full\",\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/67a85245528ba4ebea8dc539d9e23a8fe599f03e-745x480.png\",\"alt\":\"Implementation details\",\"width\":745,\"height\":480}]\n7d:[\"$\",\"p\",\"f0e9250da148\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The challenge is that engineers need and often want to move to different work over time, both for growth and motivational reasons. Deep systems"])</script><script>self.__next_f.push([1," knowledge also has value, sometimes in very specific and infrequent situations. Both forces compete with each other, and compromises around quick training or leaving docs and runbooks do not fully replace thorough understanding, even if that’s only what teams have time for. This can be supported by good engineering practices, fighting tech debt, and solid observability to make it easier to get your bearings faster, but sometimes the business will, either implicitly or explicitly, accept the risk.\"]}]\n7e:[\"$\",\"p\",\"6d6c1c22a7e7\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The principle is that the socio- parts (teams) may be more siloed than -technical parts. Isolating one without untangling the other will lead to drift, and drift to surprises. There is no easy fix for it, and incidents such as this one are great opportunities to bring some of that deep knowledge back, in a way that inherently feels practical to everyone involved.\"]}]\n7f:[\"$\",\"h2\",\"13ae865492f4\",{\"id\":\"conclusion\",\"className\":\"text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24\",\"children\":[\"Conclusion\"]}]\n80:[\"$\",\"p\",\"9037ea6970a0\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"In wanting to move past a simple explanation of “don’t destroy the Kafka cluster next time”, we have to look at a very large set of influences. This incident involved a long term response during a stressful situation, including planning, implementing, and executing a full Kafka evacuation in less than five business days. We could only accomplish this through a very rapid reorganization within Honeycomb.\"]}]\n81:[\"$\",\"p\",\"031fa594ec2e\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"We hope that this document provides a sufficient explanation behind how events unfolded for one of our biggest outages in recent memory. Documenting our response and some of its challenges can be of use for other organizations who may encounter similar situations.\"]}]\n82:[\"$\",\"p\",\"f941b4c7add3\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"An additional hope of ours is that by highlighting some of the key elements that we identified in our reviews and were in play within our broader organizational context, we can provide insights that can also help other organizations. We think the patterns uncovered do have the ability to carry meaning elsewhere.\"]}]\n83:[\"$\",\"h2\",\"bf622f64a4b6\",{\"id\":\"appendix-i:-technical-explanations\",\"className\":\"text-h3 text-hc-slate not-prose mt-6 xl:scroll-mt-24\",\"children\":[\"Appendix I: Technical Explanations\"]}]\n84:[\"$\",\"p\",\"748be3669ff3\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"This section aims to explain the mechanisms involved in a Kafka cluster. On their own, they are not considered explanations of \",[\"$\",\"em\",\"9dcfb8040938\",{\"children\":[\"why\"]}],\" the outage happened, but contributors to \",[\"$\",\"em\",\"7d49219a0ceb\",{\"children\":[\"how\"]}],\" its events unfold.\"]}]\n85:[\"$\",\"p\",\"a70806314aef\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"We assume some familiarity with the concepts described in \",[\"$\",\"$L26\",\"e41f09d5dc04\",{\"href\":\"https://docs.confluent.io/kafka/introduction.html\",\"className\":\"not-prose text-hc-cobalt mt-6 font-normal underline hover:no-underline\",\"children\":[\"Confluent's Introduction to Kafka\"]}],\".\"]}]\n86:[\"$\",\"h3\",\"ec0a852946c0\",{\"className\":\"font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24\",\"children\":[\"What are Kafka offsets?\"]}]\n87:[\"$\",\"p\",\"1461a0a12320\",{\"cla"])</script><script>self.__next_f.push([1,"ssName\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"On disk, Kafka will store data such that each message has an offset (a relative order since the start of the topic), a position (to skip around files more easily), a timestamp (to know when the data was written), and then the messages’ contents. These are put in a “log” on disk, broken down by topic and partition. Consumers can then ask to stream the log’s data, being handed the above data (including the offset and message).\"]}]\n88:[\"$\",\"p\",\"2c45cf692628\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"In practice, consumers will generally ask for an offset bigger than the last one they had seen in the stream previously. If the log gets truncated (offsets reset at 0) consumers will often just start processing again from either the start or the end of the stream.\"]}]\n89:[\"$\",\"p\",\"fabdef23bff8\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The timestamps can in theory be used to help make decisions, but are not accurate enough on their own to ensure ordering. Because clocks on computers are not reliable enough at a small enough granularity, it is possible for multiple messages to share the same timestamp, but not the same offsets.\"]}]\n8a:[\"$\",\"h3\",\"043403268619\",{\"className\":\"font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24\",\"children\":[\"Why do offsets matter to Retriever?\"]}]\n8b:[\"$\",\"p\",\"1597e55dc070\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Generally, Kafka consumers know about the offsets, but they don’t necessarily store them long term, outside of knowing where you were at.\"]}]\n8c:[\"$\",\"p\",\"d8d9c05359f7\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"For example, every time you migrate a Kafka cluster (something kind of rare), offsets get reset and lost. To make that work in complex workloads, there are migration systems that will create topics that map the new cluster’s offsets to the old cluster’s offsets (or vice versa) in order to let consumers switch from one to the other transparently.\"]}]\n8d:[\"$\",\"p\",\"0918b771b161\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Overall, the expectation is that you use offsets to track where you’re at in a topic, and if the offsets reset, you can choose to either start from 0 again, or from the newest offset.\"]}]\n8e:[\"$\",\"p\",\"67f7ba0b38d3\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Retriever is one of these systems where each bit of data we consume from Kafka gets stored again into its own column format, both on disk and on S3. When doing this, Retriever also stores the last offset seen, once globally on its own, but also within each segment metadata file it has about any data it stores.\"]}]\n8f:[\"$\",\"p\",\"2950ee2d2504\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"This is a safety mechanism that ensures Retriever never skips ahead or back in time when it shouldn’t, which could result in duplicated or missing events, which could in turn mess with investigations or alerting.\"]}]\n90:[\"$\",\"p\",\"389eabeba67c\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The problem then is that when the offsets reset on purpose (and not because of a bug),"])</script><script>self.__next_f.push([1," either due to a migration or a Kafka failure that required recreating topics or topic partitions, Retriever detects the unexpected values and \",[\"$\",\"em\",\"109ee220be44\",{\"children\":[\"refuses to process data\"]}],\" to avoid corrupting its content.\"]}]\n91:[\"$\",\"p\",\"454d41ae4c45\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"This, in turn, means that Retriever goes down hard and becomes unavailable for querying or new updates. If this happens on one partition, marking it read-only makes the rest of the cluster handle its burdens. If this happens on all partitions, no traffic can make it out of Kafka and to Retriever.\"]}]\n92:[\"$\",\"p\",\"eae0bfc61f38\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"If Retriever cannot process events within a few days (as configured by the Kafka cluster’s retention), then the messages are eventually pruned and the data is irreversibly lost.\"]}]\n93:[\"$\",\"p\",\"415ae6fe31ab\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Reducing Retriever’s dependence on offsets can either take one of two forms:\"]}]\n94:[\"$\",\"ol\",\"ae3e5c86e1cb-parent\",{\"data-testid\":\"article-list-number\",\"className\":\"text-hc-slate not-prose font-roboto mt-6 list-decimal pl-6 text-sm leading-normal font-normal tracking-normal lg:text-base\",\"children\":[[\"$\",\"li\",\"ae3e5c86e1cb\",{\"className\":\"mb-2\",\"children\":[\"make it process offsets \",[\"$\",\"em\",\"002123e0bf39\",{\"children\":[\"and\"]}],\" timestamps, such that if the offset goes back but timestamps are greater, we know the partition has reset; otherwise, follow offsets\"]}],[\"$\",\"li\",\"6888d4c7124e\",{\"className\":\"mb-2\",\"children\":[\"have Retriever able to reset the offset counts it tracks back to zero such that it will safely keep processing messages, but without overwriting old ones it has on disk or references in MySQL\"]}]]}]\n95:[\"$\",\"p\",\"e82c3ef102a8\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Both represent a significant risk to implement, because any bug can irreversibly damage and corrupt data Retriever has stored. In this incident, the evacuation work ended up implementing the second solution as a one-time operation.\"]}]\n96:[\"$\",\"h3\",\"403161f48cd9\",{\"className\":\"font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24\",\"children\":[\"What is tiered storage?\"]}]\n97:[\"$\",\"p\",\"f74beaff16ad\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Our system sees a lot of data. Tiered storage is a closed-source commercial feature of Confluent that lets Kafka split its storage into two tiers: a local one (hot) and a remote one on S3 (cold). New and fresh data that will be streamed to most live consumers is read from the hot storage, whereas older data (up to a few days old) is stored in S3.\"]}]\n98:[\"$\",\"$L2d\",\"489dfb55bdb7\",{\"priority\":\"$undefined\",\"fetchPriority\":\"low\",\"loading\":\"lazy\",\"sizes\":\"100vw\",\"data-testid\":\"article-image\",\"className\":\"not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full\",\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/8f9496198c2d829cce17f30acf4092ce357a7fa2-897x430.png\",\"alt\":\"Tiered Storage\",\"width\":897,\"height\":430}]\n99:[\"$\",\"p\",\"1689cfed8482\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"This lets us have fewer hosts with fast access for live streaming, and elastic (but slower) storage for occasionally recovering older data.\"]}]\n9a:[\"$\",\"h3\",\"bbedcddafcac\",{\"className\":\"font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium "])</script><script>self.__next_f.push([1,"lg:text-3xl xl:scroll-mt-24\",\"children\":[\"How did Kafka partitions break down?\"]}]\n9b:[\"$\",\"p\",\"bfa0ef06577d\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"At a steady state, our Kafka clusters in production have various brokers, over three availability zones, with each topic partition replicated across one broker per availability zone (AZ). Here’s an example of what this looks like with six brokers:\"]}]\n9c:[\"$\",\"$L2d\",\"9fb3ada01ddf\",{\"priority\":\"$undefined\",\"fetchPriority\":\"low\",\"loading\":\"lazy\",\"sizes\":\"100vw\",\"data-testid\":\"article-image\",\"className\":\"not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full\",\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/f4c03d079c49e23ce0b49c477c36f99d71bc41c2-994x790.png\",\"alt\":\"Steady State\",\"width\":994,\"height\":790}]\n9d:[\"$\",\"p\",\"f6843967ac02\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"In this image, you can see five topic partitions (green, blue, yellow, red, and purple), each of which touch three brokers and span three AZs. If you’re a producer of the green topic, you write to the leader (in the leftmost AZ), but can consume from any of them.\"]}]\n9e:[\"$\",\"p\",\"2a1f5b64497c\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"When an AZ fails, all brokers in that AZ die and vanish. The auto-scaling group (ASG) in AWS detects that we only have four out of six nodes, and boots two new ones in functional AZs. Once they’re up, Kafka can start replicating data there:\"]}]\n9f:[\"$\",\"$L2d\",\"77c98db48c07\",{\"priority\":\"$undefined\",\"fetchPriority\":\"low\",\"loading\":\"lazy\",\"sizes\":\"100vw\",\"data-testid\":\"article-image\",\"className\":\"not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full\",\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/704dda8b35ca3597ec26cdd32116d68742b5d6f6-1018x849.png\",\"alt\":\"AZ Failure\",\"width\":1018,\"height\":849}]\na0:[\"$\",\"p\",\"4f5d905dca5d\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"As seen here, two grayed out hosts have been added, and for a while, data will be replicated from in-sync replicas (ISRs) to those out of sync, represented by dotted lines. During that time, consumers can move to any ISR to get up-to-date data and be fine. These ISRs may be a bit more stressed, but overall, things stay up.\"]}]\na1:[\"$\",\"p\",\"0879c29499cc\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"When we killed hosts in the cleanup, \",[\"$\",\"em\",\"26122adac660\",{\"children\":[\"older\"]}],\" brokers accidentally got killed. This can further reduce redundancy, but in some unlucky cases, may fully remove all brokers that owned a topic partition:\"]}]\na2:[\"$\",\"$L2d\",\"a8c52b7110fd\",{\"priority\":\"$undefined\",\"fetchPriority\":\"low\",\"loading\":\"lazy\",\"sizes\":\"100vw\",\"data-testid\":\"article-image\",\"className\":\"not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full\",\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/b0dcbcb86dc41665fd9e40739f5e24a309cbe86b-925x586.png\",\"alt\":\"Bad Recovery\",\"width\":925,\"height\":586}]\na3:[\"$\",\"p\",\"e2f6efe1eb51\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"In this image, you can see that we killed the top two brokers (in pale red) which were in sync. Only two out of six old brokers remain, and only they have in sync data. You can see that topic partitions yellow and purple have only one healthy broker (middle-left for yellow, middle-right for purple), red was on both middle brokers and is fine, but green and blue were on neither of the remaining hosts.\"]}]\na4:[\"$\",\"p\",\"2e5364194b3a\",{\"className\""])</script><script>self.__next_f.push([1,":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"In these cases, red and blue would find themselves leaderless. The only thing telling how much data they could find would be to know how far behind the new greyed out brokers were, and whether critical metadata had the time to be replicated.\"]}]\na5:[\"$\",\"h3\",\"7b42709953fd\",{\"className\":\"font-poppins leading-heading lg:tracking-heading text-hc-slate not-prose mt-6 text-xl font-medium lg:text-3xl xl:scroll-mt-24\",\"children\":[\"Why did it only happen in prod?\"]}]\na6:[\"$\",\"p\",\"c99ae31ca9b5\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The key lies in the topology of the clusters. While the production cluster has many nodes, our pre-production clusters in the EU only have fewer nodes, in a way that makes this far less likely.\"]}]\na7:[\"$\",\"p\",\"98841c5eb940\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"Here’s an example with the smallest cluster possible, with only three nodes. When an AZ failure happens there, only one node gets killed, and once again, all topic partitions have two out of three brokers present:\"]}]\na8:[\"$\",\"$L2d\",\"f5b46dfb5a36\",{\"priority\":\"$undefined\",\"fetchPriority\":\"low\",\"loading\":\"lazy\",\"sizes\":\"100vw\",\"data-testid\":\"article-image\",\"className\":\"not-prose mt-6 max-h-[451px] rounded-[22px] block h-auto w-auto max-w-full\",\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/f20adc6769e7ba2df764868d3907c3435a8f166b-1013x780.png\",\"alt\":\"AZ Failure\",\"width\":1013,\"height\":780}]\na9:[\"$\",\"p\",\"37d78f1fe870\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"The key difference happens because we only killed one host during the failure test. Since only one was killed during the test, only one needs to be killed as part of the clean up (either of the black-outlined nodes above). By killing a single broker, we hit a total of two, and it follows that no partition will go below a single broker left, which will remain a safe leader.\"]}]\naa:[\"$\",\"p\",\"7b2e06393618\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"This means that so long as you start the AZ failure with a healthy cluster, it is difficult to break the cluster. We ran four tests in pre-production clusters, but since the issue was structurally unlikely to be reproduced there, the flaw remained hidden. On top of all the other risks of doing an AZ failover like this (database failovers, large amounts of connection interruptions across multiple services, etc.) and our frequent chaos engineering around Kafka, this made everything seem adequate on that front. With limited time and resources, other aspects of the exercise that seemed riskier were what drew the attention of people running it.\"]}]\nab:[\"$\",\"p\",\"2f4fd9bd469c\",{\"className\":\"text-hc-slate not-prose font-roboto mt-6 text-sm leading-normal font-normal tracking-normal lg:text-base xl:scroll-mt-24\",\"children\":[\"While running multiple tests in a cluster more similar to production might still not have caught the issue, the likelihood would have been much higher.\"]}]\n"])</script><script>self.__next_f.push([1,"3e:[\"$\",\"aside\",null,{\"id\":\"relatedArticles\",\"className\":\"col-span-full py-12 lg:py-24\",\"children\":[[\"$\",\"div\",null,{\"className\":\"z-20 flex flex-col items-center lg:w-1/2 lg:gap-8 mx-auto mb-0 gap-8 text-center lg:mb-0\",\"children\":[null,[\"$\",\"h1\",null,{\"_type\":\"title\",\"className\":\"font-medium text-pretty font-poppins text-center text-4xl lg:text-5xl\",\"children\":[[\"$\",\"span\",\"ca6a9642ff93d1fea2a52efc77cda8d9\",{\"className\":\"title-hero-heading text-4xl tracking-tighter lg:text-6xl\",\"children\":[\"\"]}]]}],null,null]}],[\"$\",\"div\",null,{\"className\":\"gap-4 py-10 lg:py-16 xl:flex xl:gap-6\",\"children\":[[\"$\",\"div\",null,{\"className\":\"full flex flex-col gap-4 md:flex-row lg:flex-col xl:w-[831px] xl:gap-6\",\"children\":[[\"$\",\"$L26\",\"179a5d4c-2a78-4d1f-a242-22bb1d9b4d3a\",{\"className\":\"flex-1 lg:h-[245px]\",\"href\":\"/resources/podcasts/ep-93-adaptive-sampling-strategies-with-mike-goldsmith\",\"children\":[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-50 flex h-full flex-col gap-6 rounded-4xl p-2.5 lg:flex-row\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative h-[172px] w-auto md:h-[177px] lg:h-[225px] lg:w-[396px] xl:flex-1\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/98d12fa7a1c626c805c2de91f5e19249b843b096-1080x1080.png\",\"fill\":true,\"alt\":\"O11yCast podcast logo: a dark background with a radiating orange, blue, green, and yellow circular pattern above the white and orange text '011Y_Cast(((' and 'Everything Observability'.\",\"className\":\"aspect-video rounded-4xl object-cover\",\"sizes\":\"(max-width: 768px) 100vw, 396px\"}]}],[\"$\",\"div\",null,{\"className\":\"flex flex-1 flex-col gap-4 max-lg:px-2.5 lg:pt-4\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-4\",\"children\":[[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-sky-500 text-sky-50\",\"children\":\"Podcasts\"}],[[\"$\",\"span\",null,{\"className\":\"h-3.5 border-r border-r-black/30\"}],[\"$\",\"span\",null,{\"className\":\"font-medium font-roboto text-hc-slate/70 text-xs\",\"children\":[null,[\"$\",\"span\",null,{\"children\":\"September 3, 2026\"}]]}]]]}],[\"$\",\"h3\",null,{\"className\":\"font-roboto line-clamp-2 text-base font-medium\",\"children\":\"Ep. #93, Adaptive Sampling Strategies with Mike Goldsmith\"}],[\"$\",\"p\",null,{\"className\":\"font-roboto text-hc-slate/70 line-clamp-3 text-sm font-normal max-lg:hidden\",\"children\":\"On episode 93 of O11ycast, Mike Goldsmith talks to Martin Thwaites and Ken Rimple discuss Honeycomb's contribution to the OpenTelemetry Collector, the Adaptive Tail Sampling Processor. This processor was created based on years of experience building Honeycomb's custom, industry leading tail sampler, Refinery.\\n\\n\"}]]}]]}]}],[\"$\",\"$L26\",\"a435bd91-9998-4e2e-af39-89315db18bd0\",{\"className\":\"flex-1 lg:h-[245px]\",\"href\":\"/resources/leading-with-observability/leading-with-observability-scaling-fin-to-2x-engineering-productivity\",\"children\":[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-50 flex h-full flex-col gap-6 rounded-4xl p-2.5 lg:flex-row\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative h-[172px] w-auto md:h-[177px] lg:h-[225px] lg:w-[396px] xl:flex-1\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/ca5665e8cba53cfd097fff94757ec1479ad2bf31-1920x1080.jpg\",\"fill\":true,\"alt\":\"Title slide for the series \\\"Leading With Observability,\\\" Episode 1: \\\"Scaling Fin to 2x Engineering Productivity,\\\" featuring Darragh Curran (CTO, Fin) and Charity Majors (CTO, Honeycomb).\",\"className\":\"aspect-video rounded-4xl object-cover\",\"sizes\":\"(max-width: 768px) 100vw, 396px\"}]}],[\"$\",\"div\",null,{\"className\":\"flex flex-1 flex-col gap-4 max-lg:px-2.5 lg:pt-4\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex items-center gap-4\",\"children\":[\"$Lac\",\"$Lad\"]}],\"$Lae\",\"$Laf\"]}]]}]}]]}],\"$Lb0\"]}],false]}]\n"])</script><script>self.__next_f.push([1,"ac:[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-gray-200 text-hc-slate\",\"children\":\"Leading With Observability\"}]\nad:[[\"$\",\"span\",null,{\"className\":\"h-3.5 border-r border-r-black/30\"}],[\"$\",\"span\",null,{\"className\":\"font-medium font-roboto text-hc-slate/70 text-xs\",\"children\":[null,[\"$\",\"span\",null,{\"children\":\"August 27, 2026\"}]]}]]\nae:[\"$\",\"h3\",null,{\"className\":\"font-roboto line-clamp-2 text-base font-medium\",\"children\":\"Leading With Observability: Scaling Fin to 2x Engineering Productivity\"}]\naf:[\"$\",\"p\",null,{\"className\":\"font-roboto text-hc-slate/70 line-clamp-3 text-sm font-normal max-lg:hidden\",\"children\":\"Darragh Curran, CTO at Fin (formerly Intercom), set out to double engineering productivity—and nearly tripled it instead, using AI-driven code, an AI-powered PR review system, observability as a trust mechanism, and much more hands-on leadership. In the first episode of Leading With Observability, Charity gets into the messy bits behind that transformation, not just the highlight reel.\"}]\n"])</script><script>self.__next_f.push([1,"b0:[\"$\",\"div\",null,{\"className\":\"full flex flex-col gap-4 max-xl:mt-4 md:flex-col lg:flex-row xl:flex-1 xl:flex-col xl:gap-6\",\"children\":[[\"$\",\"$L26\",\"aws-honeycomb-workshop-ai-investigators\",{\"href\":\"/resources/webinars/aws-honeycomb-workshop-ai-investigators\",\"className\":\"flex-1 lg:h-[130px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-50 flex h-full gap-4 rounded-3xl p-4 xl:gap-6 xl:rounded-4xl xl:p-2.5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative h-[84px] w-[84px] self-start xl:h-[135px] xl:w-[135px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"aspect-square h-full w-full overflow-hidden rounded-2xl bg-white xl:rounded-3xl\",\"children\":[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"viewBox\":\"0 0 542 542\",\"fill\":\"none\",\"className\":\"h-full w-full\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M0 0h542v542H0z\"}],[\"$\",\"path\",null,{\"fill\":\"#F7D2D7\",\"d\":\"M460.213 394.786c38.019-55.443 16.608-179.375-78.922-214.08-56.823-20.676-114.265-101.902-218.578-66.827a195.6 195.6 0 0 0-8.51 3.138 160.962 160.962 0 0 0-8.235 3.692C38.36 174.244 36.576 301.622 141.37 345.065c89.764 37.168 248.431 152.484 318.843 49.721Z\"}],[\"$\",\"rect\",null,{\"width\":318,\"height\":202,\"x\":107,\"y\":166,\"fill\":\"#fff\",\"rx\":5}],[\"$\",\"rect\",null,{\"width\":318,\"height\":202,\"x\":107,\"y\":166,\"stroke\":\"#E4A3A5\",\"strokeWidth\":10,\"rx\":5}],[\"$\",\"circle\",null,{\"cx\":198,\"cy\":243,\"r\":23,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M155 314a43.001 43.001 0 0 1 73.406-30.406A43.001 43.001 0 0 1 241 314h-86Z\"}],[\"$\",\"g\",null,{\"clipPath\":\"url(#a)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M301 187h47.472v47.472H301z\"}],[\"$\",\"circle\",null,{\"cx\":324.737,\"cy\":202.844,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M311.55 224.618a13.19 13.19 0 0 1 13.187-13.187 13.188 13.188 0 0 1 13.187 13.187H311.55Z\"}]]}],[\"$\",\"g\",null,{\"clipPath\":\"url(#b)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M301 243.264h47.472v47.472H301z\"}],[\"$\",\"circle\",null,{\"cx\":324.737,\"cy\":259.108,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M311.55 280.882a13.188 13.188 0 0 1 26.374 0H311.55Z\"}]]}],[\"$\",\"g\",null,{\"clipPath\":\"url(#c)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M301 299.527h47.472v47.472H301z\"}],[\"$\",\"circle\",null,{\"cx\":324.737,\"cy\":315.372,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M311.55 337.145a13.187 13.187 0 0 1 26.374 0H311.55Z\"}]]}],[\"$\",\"g\",null,{\"clipPath\":\"url(#d)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M357.264 187h47.472v47.472h-47.472z\"}],[\"$\",\"circle\",null,{\"cx\":381.001,\"cy\":202.844,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M367.814 224.618a13.188 13.188 0 0 1 13.187-13.187 13.186 13.186 0 0 1 13.187 13.187h-26.374Z\"}]]}],[\"$\",\"g\",null,{\"clipPath\":\"url(#e)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M357.264 243.264h47.472v47.472h-47.472z\"}],[\"$\",\"circle\",null,{\"cx\":381.001,\"cy\":259.108,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M367.814 280.882a13.187 13.187 0 1 1 26.374 0h-26.374Z\"}]]}],[\"$\",\"g\",null,{\"clipPath\":\"url(#f)\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#FBEDF0\",\"d\":\"M357.264 299.527h47.472v47.472h-47.472z\"}],[\"$\",\"circle\",null,{\"cx\":381.001,\"cy\":315.372,\"r\":7.053,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#E65B53\",\"d\":\"M367.814 337.145a13.186 13.186 0 1 1 26.374 0h-26.374Z\"}]]}],[\"$\",\"circle\",null,{\"cx\":412.5,\"cy\":188.5,\"r\":53.5,\"fill\":\"#E65B53\"}],[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M434.842 186.768c1.333.77 1.333 2.695 0 3.464l-32.013 18.483c-1.334.77-3-.192-3-1.732v-36.965c0-1.54 1.666-2.502 3-1.733l32.013 18.483Z\"}],[\"$\",\"path\",null,{\"fill\":\"#D75450\",\"d\":\"M238 373h55v27h-55z\"}],[\"$\",\"rect\",null,{\"width\":168,\"height\":27,\"x\":182,\"y\":400,\"fill\":\"#E4A3A5\",\"rx\":13.5}],[\"$\",\"path\",null,{\"stroke\":\"#D75450\",\"strokeLinecap\":\"round\",\"strokeWidth\":5,\"d\":\"M87.805 426.408c5.565 2.626 19.049 5.505 28.461-3.987 9.412-9.491 8.73-19.101 2.011-24.32-5.871-4.135-17.918-1.146-17.026 10.748 1.116 14.868 23.483 16.543 40.084 5.253\"}],[\"$\",\"defs\",null,{\"children\":[[\"$\",\"clipPath\",null,{\"id\":\"a\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M301 187h47.472v47.472H301z\"}]}],[\"$\",\"clipPath\",null,{\"id\":\"b\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M301 243.264h47.472v47.472H301z\"}]}],[\"$\",\"clipPath\",null,{\"id\":\"c\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M301 299.527h47.472v47.472H301z\"}]}],[\"$\",\"clipPath\",null,{\"id\":\"d\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M357.264 187h47.472v47.472h-47.472z\"}]}],[\"$\",\"clipPath\",null,{\"id\":\"e\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M357.264 243.264h47.472v47.472h-47.472z\"}]}],\"$Lb1\"]}]]}]}]}],\"$Lb2\"]}]}],\"$Lb3\",\"$Lb4\"]}]\n"])</script><script>self.__next_f.push([1,"b1:[\"$\",\"clipPath\",null,{\"id\":\"f\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M357.264 299.527h47.472v47.472h-47.472z\"}]}]\nb2:[\"$\",\"div\",null,{\"className\":\"flex flex-1 flex-col items-start gap-3 xl:pt-[6px]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-purple-100 text-purple-800\",\"children\":\"Webinars\"}],[\"$\",\"h4\",null,{\"className\":\"font-roboto line-clamp-3 text-sm font-medium xl:text-base\",\"children\":\"How Observability Makes Better AI and Human Investigators\"}]]}]\n"])</script><script>self.__next_f.push([1,"b3:[\"$\",\"$L26\",\"6dd1ee72-dedf-4b16-b7f1-a5b6f4651390\",{\"href\":\"/resources/getting-started/what-is-contextual-observability\",\"className\":\"flex-1 lg:h-[130px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-50 flex h-full gap-4 rounded-3xl p-4 xl:gap-6 xl:rounded-4xl xl:p-2.5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative h-[84px] w-[84px] self-start xl:h-[135px] xl:w-[135px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"aspect-square h-full w-full overflow-hidden rounded-2xl bg-white xl:rounded-3xl\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/a01b9544c925d87885e1c978bdee5fc2fae3250b-3840x2160.png\",\"fill\":true,\"alt\":\"What Is Contextual Observability? A Practitioner's Guide for Modern Engineering Teams\",\"className\":\"aspect-square rounded-[14px] object-cover xl:rounded-4xl\",\"sizes\":\"(max-width: 768px) 25vw, 135px\"}]}]}],[\"$\",\"div\",null,{\"className\":\"flex flex-1 flex-col items-start gap-3 xl:pt-[6px]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-green-800 text-hc-green-50\",\"children\":\"Getting Started\"}],[\"$\",\"h4\",null,{\"className\":\"font-roboto line-clamp-3 text-sm font-medium xl:text-base\",\"children\":\"What Is Contextual Observability? A Practitioner's Guide for Modern Engineering Teams\"}]]}]]}]}]\n"])</script><script>self.__next_f.push([1,"b4:[\"$\",\"$L26\",\"c81456fb-93ee-4ae9-a4b9-9d3633bb10f2\",{\"href\":\"/resources/conference-talks/socratic-ai-integrating-observability-through-interactive-dialogue-o11ycon-2026\",\"className\":\"flex-1 lg:h-[130px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"bg-hc-gray-50 flex h-full gap-4 rounded-3xl p-4 xl:gap-6 xl:rounded-4xl xl:p-2.5\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative h-[84px] w-[84px] self-start xl:h-[135px] xl:w-[135px]\",\"children\":[\"$\",\"div\",null,{\"className\":\"aspect-square h-full w-full overflow-hidden rounded-2xl bg-white xl:rounded-3xl\",\"children\":[\"$\",\"$L2d\",null,{\"src\":\"https://cdn.sanity.io/images/927dxq0h/production/8566ea5ab858dce76a0029af4067db3e6eccdd84-1920x1080.png\",\"fill\":true,\"alt\":\"Socratic AI: Integrating Observability Through Interactive Dialogue - O11yCon 2026\",\"className\":\"aspect-square rounded-[14px] object-cover xl:rounded-4xl\",\"sizes\":\"(max-width: 768px) 25vw, 135px\"}]}]}],[\"$\",\"div\",null,{\"className\":\"flex flex-1 flex-col items-start gap-3 xl:pt-[6px]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"font-roboto focus:ring-ring inline-flex items-center rounded-lg font-medium transition-colors focus:ring-2 focus:ring-offset-2 focus:outline-none px-2 py-1 text-xs leading-normal bg-hc-gold-300 text-gold-1000\",\"children\":\"Conference Talks\"}],[\"$\",\"h4\",null,{\"className\":\"font-roboto line-clamp-3 text-sm font-medium xl:text-base\",\"children\":\"Socratic AI: Integrating Observability Through Interactive Dialogue - O11yCon 2026\"}]]}]]}]}]\n"])</script></body></html> |