Files
nexus/sreweekly/articles/294/07-incident-review-and-postmortem-best-practices.html
2026-09-12 17:23:01 +08:00

486 lines
269 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8" />
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
<link rel="preconnect" href="https://substackcdn.com" />
<title data-rh="true">Incident Review and Postmortem Best Practices</title>
<meta data-rh="true" name="theme-color" content="#ffffff"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Incident Review and Postmortem Best Practices"/><meta data-rh="true" name="twitter:title" content="Incident Review and Postmortem Best Practices"/><meta data-rh="true" name="description" content="A survey of how companies deal with incidents today, and a peek into the best practices of the future."/><meta data-rh="true" property="og:description" content="A survey of how companies deal with incidents today, and a peek into the best practices of the future."/><meta data-rh="true" name="twitter:description" content="A survey of how companies deal with incidents today, and a peek into the best practices of the future."/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!sJy5!,w_1200,h_675,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!TSX_!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fpragmaticengineer.substack.com%2Fapi%2Fv1%2Fpost_preview%2F42747309%2Ftwitter.jpg%3Fversion%3D4"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" property="article:modified_time" content="2025-11-07T15:27:15.404Z"/>
<style>
@layer legacy, pencraftReset, pencraft;
</style>
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.cee9fbee.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/86118.c25b9f67.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/76768.f15fc605.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/41932.97d2663c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/32552.313415d5.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
<meta name="author" content="Gergely Orosz" />
<meta property="og:url" content="https://newsletter.pragmaticengineer.com/p/incident-review-best-practices" />
<link rel="canonical" href="https://newsletter.pragmaticengineer.com/p/incident-review-best-practices" />
<link rel="shortcut icon" href="https://substackcdn.com/image/fetch/$s_!ZSdd!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Ffavicon.ico">
<link rel="icon" type="image/png" sizes="16x16" href="https://substackcdn.com/image/fetch/$s_!shD_!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Ffavicon-16x16.png">
<link rel="icon" type="image/png" sizes="32x32" href="https://substackcdn.com/image/fetch/$s_!Cig7!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Ffavicon-32x32.png">
<link rel="icon" type="image/png" sizes="48x48" href="https://substackcdn.com/image/fetch/$s_!dMyM!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Ffavicon-48x48.png">
<link rel="apple-touch-icon" sizes="57x57" href="https://substackcdn.com/image/fetch/$s_!TlPE!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="https://substackcdn.com/image/fetch/$s_!ktEI!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="https://substackcdn.com/image/fetch/$s_!8TUl!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="https://substackcdn.com/image/fetch/$s_!Ij57!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="https://substackcdn.com/image/fetch/$s_!9FlB!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="https://substackcdn.com/image/fetch/$s_!fmrL!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="https://substackcdn.com/image/fetch/$s_!RHaf!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="https://substackcdn.com/image/fetch/$s_!7yNl!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-152x152.png">
<link rel="apple-touch-icon" sizes="167x167" href="https://substackcdn.com/image/fetch/$s_!I-zz!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-167x167.png">
<link rel="apple-touch-icon" sizes="180x180" href="https://substackcdn.com/image/fetch/$s_!iIpB!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-180x180.png">
<link rel="apple-touch-icon" sizes="1024x1024" href="https://substackcdn.com/image/fetch/$s_!NoaE!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F47d77fcc-2914-46a1-8d39-3a6e8758e0ad%2Fapple-touch-icon-1024x1024.png">
<link rel="alternate" type="application/rss+xml" href="/feed?sectionId=132248" title="The Pragmatic Engineer"/>
<style>
@font-face{font-family:'Lora';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QI8MX1D_JOuMw_hLdO6T2wV9KnW-MoFoqJ2nPWc3ZyhTjcV.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Lora';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QI8MX1D_JOuMw_hLdO6T2wV9KnW-MoFoqt2nPWc3ZyhTjcV.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Lora';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QI8MX1D_JOuMw_hLdO6T2wV9KnW-MoFoqB2nPWc3ZyhTjcV.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Lora';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QI8MX1D_JOuMw_hLdO6T2wV9KnW-MoFoqF2nPWc3ZyhTjcV.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Lora';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QI8MX1D_JOuMw_hLdO6T2wV9KnW-MoFoq92nPWc3ZyhTg.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Lora';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwf7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Lora';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMw77I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Lora';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwX7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Lora';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwT7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Lora';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwr7I_FMl_E.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Lora';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwf7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Lora';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMw77I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Lora';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwX7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Lora';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwT7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Lora';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwr7I_FMl_E.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Lora';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwf7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Lora';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMw77I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Lora';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwX7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Lora';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwT7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Lora';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwr7I_FMl_E.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Lora';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwf7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Lora';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMw77I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Lora';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwX7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Lora';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwT7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Lora';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwr7I_FMl_E.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
</style>
<style data-pub-theme-prefix>:root{--color_theme_bg_pop:#FF6B00;--background_pop:#FF6B00;--cover_bg_color:#FFFFFF;--cover_bg_color_secondary:#f0f0f0;--background_pop_darken:#e66000;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#e66000;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(255, 107, 0, 0.2);--color_theme_bg_pop_30:rgba(255, 107, 0, 0.3);--print_pop:#ff6b00;--color_theme_accent:#ff6b00;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#ff6b00;--font_family_headings_preset:Lora,sans-serif;--font_weight_headings_preset:600;--font_family_body_preset:'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';--font_weight_body_preset:400;--font_preset_heading:fancy_serif;--font_preset_body:sans;--home_hero:feature-media;--home_posts:list;--web_bg_color:#ffffff;--background_contrast_1:#f0f0f0;--background_contrast_2:#dddddd;--background_contrast_3:#b7b7b7;--background_contrast_4:#929292;--background_contrast_5:#515151;--color_theme_bg_contrast_1:#f0f0f0;--color_theme_bg_contrast_2:#dddddd;--color_theme_bg_contrast_3:#b7b7b7;--color_theme_bg_contrast_4:#929292;--color_theme_bg_contrast_5:#515151;--color_theme_bg_elevated:#ffffff;--color_theme_bg_elevated_secondary:#f0f0f0;--color_theme_bg_elevated_tertiary:#dddddd;--color_theme_detail:#e6e6e6;--background_contrast_pop:rgba(255, 107, 0, 0.4);--color_theme_bg_contrast_pop:rgba(255, 107, 0, 0.4);--theme_bg_is_dark:0;--print_on_web_bg_color:#363737;--print_secondary_on_web_bg_color:#868787;--background_pop_rgb:255, 107, 0;--color_theme_bg_pop_rgb:255, 107, 0;--color_theme_accent_rgb:255, 107, 0;}</style>
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<style data-pub-theme-postfix></style>
<script async="true" src="https://www.googletagmanager.com/gtag/js?id=G-SL4ERK2LEQ&l=GA4DataLayer" >
</script>
</head>
<body class="">
<div id="entry">
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:71px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="flex-basis:0px;flex-grow:1;" class="logoContainer-p12gJb"><a href="/" native class="pencraft pc-display-contents pc-reset"><div draggable="false" class="pencraft pc-display-flex pc-position-relative pc-reset"><div style="width:40px;height:40px;" class="pencraft pc-display-flex pc-reset bg-white-ZBV5av pc-borderRadius-sm overflow-hidden-WdpwT6 sizing-border-box-DggLA4"><picture class="pencraft pc-display-contents pc-reset"><source type="image/webp" sizes="40px" srcset="https://substackcdn.com/image/fetch/$s_!eod5!,w_40,h_40,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png 40w, https://substackcdn.com/image/fetch/$s_!eod5!,w_80,h_80,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png 80w, https://substackcdn.com/image/fetch/$s_!eod5!,w_120,h_120,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png 120w" style="display:none;"/><img style="object-fit:cover;width:40px;height:40px;" src="https://substackcdn.com/image/fetch/$s_!eod5!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png" srcset="https://substackcdn.com/image/fetch/$s_!eod5!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png 40w, https://substackcdn.com/image/fetch/$s_!eod5!,w_80,h_80,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png 80w, https://substackcdn.com/image/fetch/$s_!eod5!,w_120,h_120,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png 120w" sizes="40px" alt="The Pragmatic Engineer" draggable="false" class="pencraft pc-reset"/></picture></div></div></a></div><div style="flex-grow:0;" class="titleContainer-DJYq5v"><h1 class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd"><a href="/" class="pencraft pc-display-contents pc-reset">The Pragmatic Engineer</a></h1></div><div style="flex-basis:0px;flex-grow:1;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Fincident-review-best-practices%3Fs%3Dw&amp;for_pub=pragmaticengineer" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="height:72px;"></div></div></div><div><script nonce="prTDrEptRhTGJ_GBwV-4sg" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://newsletter.pragmaticengineer.com/p/incident-review-best-practices","mainEntityOfPage":"https://newsletter.pragmaticengineer.com/p/incident-review-best-practices","headline":"Incident Review and Postmortem Best Practices","description":"A survey of how companies deal with incidents today, and a peek into the best practices of the future.","image":[{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!sJy5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png"}],"datePublished":"2021-10-19T15:14:03+00:00","dateModified":"2021-10-19T15:14:03+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"Gergely Orosz","url":"https://substack.com/@pragmaticengineer","description":"Big Tech and startups from the inside. Especially relevant for software engineers / AI engineers, useful for anyone working in tech.","identifier":"user:30107029","sameAs":["https://twitter.com/GergelyOrosz"],"image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!CPFa!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!CPFa!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png"}}],"publisher":{"@type":"Organization","name":"The Pragmatic Engineer","url":"https://newsletter.pragmaticengineer.com","description":"Big Tech and startups, from the inside. Highly relevant for software engineers, AI engineers and engineering leaders, useful for those working in tech.","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":1000000},"identifier":"pub:458709","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!eod5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!eod5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!eod5!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!eod5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!eod5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!eod5!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png"},"sameAs":["https://twitter.com/GergelyOrosz"]},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":101},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":3},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":2}]}</script><script nonce="prTDrEptRhTGJ_GBwV-4sg" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"The Pragmatic Engineer","item":"https://newsletter.pragmaticengineer.com"},{"@type":"ListItem","position":2,"name":"Incident Review and Postmortem Best Practices","item":"https://newsletter.pragmaticengineer.com/p/incident-review-best-practices"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><div class="pencraft pc-display-flex pc-reset post-label"><a href="https://newsletter.pragmaticengineer.com/s/deepdives/?utm_source=substack&amp;utm_medium=menu" class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-14-MLPa7j weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA">Deepdives</a></div><h1 dir="auto" class="post-title published title-X77sOw">Incident Review and Postmortem Best Practices</h1><h3 dir="auto" class="subtitle subtitle-HEEcLo">A survey of how companies deal with incidents today, and a peek into the best practices of the future.</h3><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-row pc-gap-8 pc-alignItems-center pc-justifyContent-flex-start pc-reset"><div style="--scale:36px;--offset:9px;--border-width:4.5px;" class="pencraft pc-display-flex pc-flexDirection-row pc-alignItems-center pc-justifyContent-flex-start pc-reset ltr-qDBmby"><a href="https://substack.com/@pragmaticengineer" aria-label="View Gergely Orosz's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:36px;" tabindex="0" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6 avatar-u8q6xB last-JfNEJ_"><div style="--scale:36px;" title="Gergely Orosz" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!CPFa!,w_36,h_36,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png 1x, https://substackcdn.com/image/fetch/$s_!CPFa!,w_72,h_72,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png 2x, https://substackcdn.com/image/fetch/$s_!CPFa!,w_108,h_108,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!CPFa!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png" alt="Gergely Orosz's avatar" srcset="https://substackcdn.com/image/fetch/$s_!CPFa!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png 1x, https://substackcdn.com/image/fetch/$s_!CPFa!,w_72,h_72,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png 2x, https://substackcdn.com/image/fetch/$s_!CPFa!,w_108,h_108,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png 3x" width="36" height="36" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-reset color-primary-zABazT line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><span style="min-width:0;" data-state="closed"><a href="https://substack.com/@pragmaticengineer" class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ">Gergely Orosz</a></span></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2021-10-19T15:14:03.962Z" title="2021-10-19T15:14:03.962Z">Oct 19, 2021</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (101)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">101</div></button></div><button tabindex="0" type="button" aria-label="View comments (2)" data-href="https://newsletter.pragmaticengineer.com/p/incident-review-best-practices/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg><div class="label">2</div></button><button tabindex="0" type="button" id="radix-P0-27" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">3</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p><em><span>👋 Hi, this is </span><a href="https://twitter.com/gergelyorosz">Gergely</a><span> with this month’s free edition of the Pragmatic Engineer Newsletter. In every issue, I cover challenges at big tech and high-growth startups through the lens of engineering managers and senior engineers.</span></em></p><p><em>Subscribe to get weekly issues. Many subscribers expense this newsletter to their learning and development budget.👇</em></p><div data-component-name="SubscribeWidget" class="subscribe-widget"><div class="pencraft pc-display-flex pc-justifyContent-center pc-reset"><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe-widget"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><input type="hidden" name="source_post_id" value="42747309"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div><p></p><p><span>One reason incidents are important is that they often reveal the </span><em>real</em><span> state of products, teams or organizations, which is often very different from the imaginary picture that engineering leaders have in their heads. Transparent incident reports and a good incident-handling strategy can inject much-needed realism into the development process. It’s hard to brush aside incidents that have caused specific damage, indeed these incidents are often powerful ways to make cases for work that would otherwise be delayed indefinitely.</span></p><p>The only certain thing about outages is that they will always happen. Everything else is up to us. How much effort do we put into preventing them, or into mitigating them quickly, or learning from them?</p><p>I’ve talked with dozens of engineers at a variety of companies and have found that no one is fully happy with how they are handling incidents. </p><p>In this issue we cover:</p><ul><li><p><strong>Common incident handling practices across the industry. </strong><span>What are the trends on how tech companies approach incidents today?</span></p></li><li><p><strong>Incident review best practices</strong><span>. What are processes, tools and approaches that we can point to as sensible practices?</span></p></li><li><p><strong>Incident review practices of tomorrow</strong><span>. A few teams and companies have moved beyond what we’d call the best practices of today. What is their approach and how is it working?</span></p></li><li><p><strong>What tech can learn from incident handling in other industries.</strong><span> Incidents are not unique to tech; fields like healthcare, the military and many others have a long history of efficiently dealing with incidents. What can we learn from them?</span></p></li><li><p><strong>Incident review/postmortem examples and templates.</strong><span> A selection of case studies you can get inspiration from and 🔒</span><a href="https://newsletter.pragmaticengineer.com/p/templates-as-inspiration-for-engineering">subscriber-only postmortem templates</a><span>.</span></p></li></ul><p>The oncall process – monitoring and alerting – is beyond the scope of this article. We cover what happens starting from when an outage or incident is confirmed.</p><p><em>Note: This article mentions vendors specializing in incident handling-related services. I have not been paid to mention these companies. My newsletter is fully independent, does not offer sponsorships and I have no commercial affiliation with the vendors mentioned.</em></p><h2 class="header-anchor-post">How Companies Handle Incidents<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§how-companies-handle-incidents" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.pragmaticengineer.com/i/42747309/how-companies-handle-incidents" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><span>More than 60 teams have shared information on how they respond to outages, and what happens afterwards. </span><strong><a href="https://docs.google.com/spreadsheets/d/1GPINipdf-l2H05iKOUbpkrqwlZ61ZCJDnwY5iE8LtRM/edit?usp=sharing">See aggregated survey results here.</a></strong></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!sJy5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!sJy5!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 424w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 848w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 1272w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!sJy5!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png" width="1456" height="690" data-attrs="{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:690,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:1769147,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!sJy5!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 424w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 848w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 1272w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a><figcaption class="image-caption">How teams respond to outages - some of the results.</figcaption></figure></div><p>98.5% of companies sharing details have an incident management process in place. The only exception was a Series C property management company which used an ad-hoc process on top of email, to manage incidents. I was expecting more variation on the process, the tooling and the steps, but there were large overlaps across companies. This might be explained by selection bias, meaning the survey was mostly filled out by engineers whose teams already take incidents seriously.</p><p>The majority of companies sharing data follow a process that can be summarized like this:</p><ul><li><p><strong>An outage is detected. </strong><span>This could happen following an automated alert, or someone noticing that something is “off”.</span></p></li><li><p><strong>An incident is declared.</strong><span> An engineer – often the on-call engineer – confirms that an outage is happening. They follow an agreed process to declare this incident, which is often a mix of declaring it over chat, and sometimes opening a ticket in a system used to track incidents.</span></p></li><li><p><strong>The incident is being mitigated.</strong><span> During mitigation, the engineers involved provide updates. Some companies have more processes around this than others; for example, making it clear who the Incident Leader or Incident Commander is, or expecting regular updates to stakeholders when high-priority outages happen.</span></p></li><li><p><strong>The incident has been mitigated.</strong><span> Everyone breathes a sigh of relief.</span></p></li><li><p><strong>Decompression period.</strong><span> There is rarely an expectation for engineers to immediately kick off root cause investigation. This is especially true when the mitigation happened outside business hours, say in the middle of the night. However, many companies have a target window to complete analysis within 36-48 hours, which puts pressure on engineers to shorten this decompression time.</span></p></li><li><p><strong>Incident analysis / postmortem / root cause analysis</strong><span>. Almost every company calls the investigation that follows by a different name. The idea is the same, the people involved in the incident get together one way or the other, and summarize what happened, and what the learnings are.</span></p></li><li><p><strong>Incident review</strong><span>. For outages with a large enough impact, a bigger group reviews the analysis put together in the previous step. Some companies have dedicated incident management teams running this process, some have weekly or bi-weekly meetings with some managers present, while others do it ad hoc.</span></p></li><li><p><strong>Incident actions tracked for follow up.</strong><span> In the analysis and review, teams come up with improvements on how to either prevent similar outages, reduce the time to mitigate, or the impact of the outage. During the review, these actions are captured, and added to the roadmap or backlog of the teams owning these.</span></p></li></ul><p>What tooling do teams use when handling incidents? Based on survey responses, these were the most common mentions:</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!gTi-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!gTi-!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 424w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 848w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 1272w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!gTi-!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png" width="1254" height="624" data-attrs="{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:624,&quot;width&quot;:1254,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:111147,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!gTi-!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 424w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 848w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 1272w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a><figcaption class="image-caption">Common tooling teams use when handling incidents, based on survey results.</figcaption></figure></div><p><strong>Less common approaches</strong><span> that a few companies mentioned:</span></p><ul><li><p><em>Research the same day, postmortem the next day.</em><strong> </strong><span>“Our team does a little research right afterwards, while the incident is still fresh. We then start the postmortem process the next day. It’s down to the engineers to decide how they go about it; the point is they don’t need to start the postmortem right away.”</span></p></li><li><p><em>Dedicated decompression time.</em><span> “The team which owns the service/system that failed takes time out from regular work to decompress (if it was a big outage) and to work on post-incident review and immediate follow-up actions.”</span></p></li><li><p><em>Create low priority incidents to broadcast risky migrations. Some companies with incident systems that are used to check the health of systems, often create low-priority incidents to raise awareness for risky migrations that could result in temporary issues, but which they are actively monitoring for.</em></p></li></ul><h2 class="header-anchor-post">Incident Handling Best Practices<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§incident-handling-best-practices" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.pragmaticengineer.com/i/42747309/incident-handling-best-practices" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>What are practices that stand out in their effectiveness, when we think of incidents? </p><ul><li><p><strong>Encourage raising incidents, even when in doubt.</strong><span> It can be stressful to decide whether or not an incident should be created when something looks “off”. At the same time, many incidents are either delayed in their response, or are missed completely because someone did not ring the alarm when they had a nagging suspicion. Create a culture where it’s encouraged to raise an incident. Even if it turns out to not be an outage, it’s worthwhile learning to investigate </span><em>why</em><span> it felt like something was wrong.</span></p></li><li><p><strong>Be clear on roles during incidents</strong><span>. Make it clear who is in charge of mitigating the incident, and who should communicate with stakeholders. Many places call this role the Incident Commander. The role of leading mitigation and communicating might be the same. However, for high severity outages, or when many stakeholders need to be kept in on how it’s going, it might make sense to separate them.</span></p></li><li><p><strong>Define incident severity levels ahead of time</strong><span>. Have a shared understanding of what makes an outage more ‘important’. Write down the definition, and use the same definitions across the organization. Most companies define severity levels based on how customers might be impacted, and the impact radius based on what percentage of these customers are impacted. Over time, you’ll probably need to update your definitions as new types of incidents occur.</span></p></li><li><p><strong>Have playbooks ready </strong><span>for common incidents, and make them easy to locate and use. Playbooks could be referenced as part of the alert, or listed at a place where it’s easy for the oncall engineer to find them. Playbooks help with onboarding new members to the rotation, while knowing what to check for – and in what order – can help mitigate outages faster.</span></p></li><li><p><strong>Leave time for engineers to work on an incident review/postmortem</strong><span>. While it’s always possible to rush this process, you’ll get lower quality reviews if you do so. Instead, give people the space to get together and discuss what happened, and aim for all those involved to have their say.</span></p></li><li><p><strong>Dig deeper when looking into root causes of the incident.</strong><span> Don’t stop at the first and often obvious, potential reason. Ask questions to understand how various factors could have contributed to the outage, and what are the things that contributed to those factors.</span></p></li><li><p><strong>Share incident analysis and learnings with a broader group</strong><span>, instead of keeping it within the small group which reviews incidents.</span></p></li><li><p><strong>Find and use tools built for efficient incident handling.</strong><span> There are companies whose incident review tooling is email-based, and even a simple approach like this can work well enough. However, if you use tools that support the understanding of common incident workflows, you’ll see less confusion, and less tribal knowledge being needed to handle outages. Many companies build their own tooling, and there’s a world of vendors and open source services teams can use. Contributors to this article include engineers from incident handling tools:</span></p><ul><li><p><a href="https://incident.io/?utm_source=newsletter&amp;utm_medium=email&amp;utm_campaign=pragmatic_engineer">incident.io</a><span> (dedicated incident response tools)</span></p></li><li><p><a href="https://www.jeli.io/?utm_source=newsletter&amp;utm_medium=email&amp;utm_campaign=pragmatic_engineer">Jeli.io</a><span> (incident analysis platform)</span></p></li><li><p><a href="https://www.blameless.com/?utm_source=newsletter&amp;utm_medium=email&amp;utm_campaign=pragmatic_engineer">Blameless.com</a><span> (end-to-end platform). </span></p></li><li><p>Do your research: you’ll find plenty of others as well.</p></li></ul></li></ul><p><strong>Blameless reviews/postmortems </strong><span>are worth talking more about. When doing a root cause analysis, avoid making it seem like a single person is responsible for the incident. Most outages will be caused by configuration or code changes that someone did, and it’s very easy to find out who it was. However, it could have been </span><em>anyone</em><span> else on the team. </span></p><p>Instead of directly or indirectly putting the blame on one person, go further and look at why systems allowed anyone to make those changes, without providing feedback. If the conditions that allowed the incident to happen are unaddressed, these conditions could trip up someone else on the team in future.</p><p>Some people resist the idea of “pushing” the use of blameless postmortems across an organization. “Will this not lead to a lack of accountability?” they ask.</p><p>But accountability and a blameless culture are two separate things in my view, which can – and do – go hand in hand. Accountability means people take responsibility for their work, and when things inevitably go wrong, they take ownership of fixing things. A blameless approach recognizes and embraces that it’s counterproductive to fault someone for doing something they were unaware would cause an issue. This is especially the case when those people take accountability for addressing the reasons that led to the issue.</p><p><strong>Well-run incident review meetings</strong><span> are key to sharing context with all involved, and to coming to better learnings. Here is a structure that some of the best run review discussions followed:</span></p><ul><li><p><em>Do the work upfront. </em><span>By the time of a meeting, the people involved in the outage should have come together, discussed the incident and their take on it, and put together resources that can be used to give others context. </span></p></li><li><p><em>Level set in the beginning.</em><span> Don’t dive in straight to the incident. Start with sharing high-level details and context, so everyone in the room gets on the same page.</span></p></li><li><p><em>Walk and talk through the timeline as it happened. </em><span>Don’t just run a script, but talk through what was happening. What looked weird? Why did you have a nagging concern? If you’re in a review, jump in and ask questions. Understanding how things unfolded is much more important than most people realize.</span></p></li><li><p><em>Keep it blameless.</em><span> Avoid referring to any one person being the one “to blame” for the incident.</span></p></li><li><p><em>Recognize what went well, and what can be improved. Don’t focus only on what’s lacking, but call out actions, processes, systems that worked well, some of which might have been learnings from another incident.</em></p></li><li><p><em>Track actions as they come up. As ideas come up for things that can be changed or improved, keep a running list of these.</em></p></li><li><p><em>Consider separating reviews and action items meetings. By focusing on &quot;what can we learn&quot; in the first review meeting, much better improvement actions emerge in a second meeting focused on action item identification, held a day or more later.</em></p></li><li><p><em>Record the meeting</em><span> so those who missed it can catch up, and have the full context.</span></p></li><li><p><em>Open up the meeting to a wider group,</em><span> so people with additional context, or those curious can join. This practice can help with transparency and spreading of information.</span></p></li></ul><p><span>I sat down with </span><a href="https://twitter.com/evnsio">Chris Evans</a><span>, co-founder and CPO at incident.io, formerly head of platform at Monzo. I asked him which other practices he’s seen adopted by engineering teams which are great at incident handling. Here’s what he shared:</span></p><ul><li><p><strong>Avoid “blameless buck-passing”. </strong><span>When a high severity outage happens, there is often pressure from above on the team to “do everything so this can never ever happen again.” This approach, in turn, can turn reviews into blameless buck passing exercises, in which those involved try to shift any implied blame away from them or their team, to somewhere else. On the surface, it’s a blameless process. In reality, it’s a poor experience for everyone involved and creates unnecessary pressure.</span></p></li><li><p><strong>Remove the pressure</strong><span> of it “never happening again” and of all reviews needing to have action items. Do these two things and you’ll have far more effective incident reviews.</span></p></li><li><p><strong>Give “soak time” after incidents</strong><span>. After the outage, leave time for the team to talk and soak up what really happened.</span></p></li><li><p><strong>Think of incident reviews as a process, not as an artifact.</strong><span> Too many teams focus on writing the artifact, basically a report, instead of the review being a process that starts before you write anything down, and does not end when a document is complete.</span></p></li><li><p><strong>Treat an incident as an “aha moment”.</strong><span> Every incident sheds light on something the team did not know before. What was it?</span></p></li><li><p><strong>Consider retiring the title “postmortem”.</strong><span> Postmortem means examining a body to determine the cause of death. This has a grim association and starts with the assumption that something terrible has happened.</span></p></li></ul><h2 class="header-anchor-post">Beyond the Best Practices of Today<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§beyond-the-best-practices-of-today" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.pragmaticengineer.com/i/42747309/beyond-the-best-practices-of-today" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><span>Observability platform </span><a href="https://www.honeycomb.io/?utm_source=newsletter&amp;utm_medium=email&amp;utm_campaign=pragmatic_engineer">Honeycomb</a><span> stood out in their responses to my survey. They were one of the few firms which are moving away from templates and do not track action items. Unlike most other teams which want to do this, they actually took the step.</span></p><p><span>From many other teams, I would have interpreted this as trying out something that might or might not work. However, Honeycomb handles huge amounts of data, and provides stricter Service Level Agreements (SLAs) than most of their competitors. They also take reliability very seriously, so much that a 5-minute delay in data processing is </span><a href="https://twitter.com/mipsytipsy/status/1443020069483651073?s=20">an outage they publicly report</a><span>.</span></p><p><span>I asked Honeycomb engineer </span><a href="https://twitter.com/paulosman">Paul Osman</a><span> why they made this shift, and how it’s working out. Paul shared how two years ago, they noticed that action items coming up during reviews were not particularly interesting findings; they were mostly things the team was going to do, anyway.</span></p><p><strong>Focusing on the learning during incident reviews, over explicitly tracking action items,</strong><span> has been the biggest shift, Paul shared. Teams and individual engineers still create their own tasks, not just for incident reviews, but for everyday work. They’ll also create those tasks even if there’s not an incident review. Paul said:</span></p><p>‘For us, the review is more about “who knew what when and how did they know it?” and “how did our systems surprise us?” instead of “what action items can we extract from this?”</p><p>‘Most of the interesting insights we’ve found are in a different category from tasks. They’re things like “new traffic patterns can show up as red herrings when debugging” or “Hey, this person is needed whenever we have a problem with this system, we should schedule someone to pair with them so they can go on vacation”. </p><p>Honeycomb have left behind some practices that outlived their usefulness, and seem to be better off for it. However, I’d add that they’re still a relatively small group – around 35 engineers – at a place where everyone operates with high autonomy.</p><p><strong>Moving on from the Five Whys method</strong><span> for gathering more information,</span><strong> </strong><span>is another approach used by teams which are great at incident handling. The Five Whys is still considered as a best practice by many teams and is a common way to run the root-cause analysis process.</span><strong> </strong><span>The idea here is to ask “why” in succession, going deeper and uncovering more information each time.</span></p><p><span>The framework is very easy to get started, when teams don’t do much digging into incidents. However, as Andrew Hatch at LinkedIn shares in the talk </span><a href="https://www.usenix.org/conference/srecon21/presentation/hatch">Learning More from Complex systems</a><span>, there are risks to relying on the Five Whys:</span></p><blockquote><p><em><span>The danger of the Five Whys is how, by following it, we might miss out on other root causes of the incident. (...) </span><strong>We’re not broadening our understanding.</strong><span> We’re just trying to narrow down on one thing, fix it, and hope that this will make the incident not happen again.</span></em></p></blockquote><p><span>John Allspaw also advocates against using the Five Whys in his </span><a href="https://www.kitchensoap.com/2014/11/14/the-infinite-hows-or-the-dangers-of-the-five-whys/">The Dangers of The Five Whys</a><span> article. He says that asking “why” is a sure way to start looking for a “who”, and to start looking for who is to blame for the outage.</span></p><p><span>Instead of asking “why?” multiple times, consider asking “how did it make sense for someone to do what they did?”. Then dig deeper in gathering the context that made the choices logical, in the context of the people acting. Here are </span><a href="http://www.kitchensoap.com/wp-content/uploads/2014/09/Velocity2014-PM-Fac-Handout-Debrief.pdf">some other questions to consider asking</a><span> to run healthy debriefs. </span></p><p><strong>Take a socio-technical systems approach to understand the outage.</strong><span> A root cause analysis or the Five Whys approach can be appealing because they simplify a post-incident effort, but they will not address the deeper systemic issues within your system, says </span><a href="https://twitter.com/LauraMDMaguire">Laura Maguire</a><span>, Head of Research at Jeli.io. </span></p><p>Socio-technical systems approaches look at both the technical aspects – what broke – and the ‘social’ aspects; how the incident was handled. This gives an oncall team deeper information about software component failures. It also surfaces relevant organizational factors, such as who has specialized knowledge about different aspects of the system and which stakeholders need updates at what frequencies, in order to minimize impacts to their areas of the business. Some guidance for conducting a systemic investigation:</p><ul><li><p><em>Use human error as a starting point for an investigation</em><span>. For example, if an engineer appears to have made a mistake, look at the processes they used, the training they received, how frequently they interact with that part of the system or codebase, their workload, their oncall schedule and other relevant details that put an error into the broader context. Sidney Dekker, founder of the Safety Science Innovation Lab advocates for this approach in the book, </span><a href="https://www.amazon.com/Field-Guide-Understanding-Human-Error/dp/1472439058">The Field Guide to Understanding 'Human Error'</a><span>.</span></p></li><li><p><em>Consider organizational factors</em><span>. Do teams have a headcount that allows them to execute on their roles and responsibilities in a manner that isn’t burning people out? Are there legacy systems that haven’t been dealt with? Is ownership of services clear? </span></p></li><li><p><em>Gather contextual information relevant to the event</em><span>. Questions such as “Were you on call?”, “How did you get brought into the incident”, “Have you worked with this team before?” can reveal important information about the ad hoc or informal ways teams adapt to cope with unusual or difficult outages. </span></p></li><li><p><em>Encourage participants to recount their experiences as they happened</em><span>, not as they think they </span><em>should </em><span>have responded in hindsight. Focusing on what happened can surface partial or incomplete mental models about how the system works. It can reveal new ways of approaching a problem or monitoring anomalous behaviour and can help a team share specialized knowledge. If psychological safety is present, sharing these stories publicly can allow others to hear what may not be apparent to everyone else. </span></p></li><li><p><em>A systemic investigation is about asking a different set of questions</em><span> and recognizing that individual actions and decisions are influenced and constrained by the system of work they take place in.</span></p></li></ul><h2 class="header-anchor-post">Incident Review Practices of Tomorrow<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§incident-review-practices-of-tomorrow" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.pragmaticengineer.com/i/42747309/incident-review-practices-of-tomorrow" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><span>I sat down to talk about the future of incident management with </span><a href="https://twitter.com/allspaw">John Allspaw</a><span>, who has been heavily involved in this space for close to a decade. He was engineer #9 at Flickr at the time of Flickr’s massive growth phase. He then became the CTO of Etsy, where he worked for seven years. </span></p><p><span>For the past decade, John has been going deeper and deeper into how to build better resilient systems. He enrolled in Lund University in Sweden in the Human Factors and Safety program, a program where some of the leading thinkers in resilience engineering seem to have crossed paths. He has founded </span><a href="https://www.adaptivecapacitylabs.com/">Adaptive Capacity Labs</a><span>, which partners with organizations that want to further improve how they handle and learn from incidents. </span></p><p>As we talked for an hour, I kept being surprised by the depth of his understanding of resilient systems. It slowly made sense why he spent so much time studying non-tech related subjects, all connected to resilient engineering systems. John also exposed a world of decades-old research outside tech that I would have never thought to look to as inspiration to build better, more reliable systems, but which perhaps we all should. Here’s a summary of our conversation. My questions are in italics.</p><p><em>Why have you been spending so much time on the incident space?</em></p><p><strong>‘Incidents create attention energy </strong><span>around them. When something goes wrong, people pay far more attention to everything around the event than when it’s business as usual. This is also why incidents and outages can be a catalyst to kicking off larger changes within any organization, not just tech companies.’</span></p><p><em>What is your view on industry best practices which many companies follow, like templates, incident reviews, follow-up items?</em></p><p>‘Incident handling practices in the industry are well-intentioned, and point in the right direction. However, these practices are often poorly calibrated. We talk a lot about learning from incidents, and some learning is certainly happening. However, it’s not happening as efficiently as it could.</p><p><strong>‘We often confuse fixing things fast with learning. </strong><span>Take the incident review that most companies follow. There’s an hour, at most, to go through multiple incidents. There is often more focus on generating follow-up items than learning from the incident. In fact, most people seem to think that by generating follow-up items, learning will also happen. However, this is far from the case.</span></p><p>‘When we talk with companies, we ask them to describe how they handle incidents. They jump in, talk about how they detect outages, how they respond, who plays what role, and which tools they use. They’ll often mention cliches like “we never let a good incident go to waste.”</p><p>‘However, most teams cannot describe a major incident in detail. We typically ask them to talk us through a specific incident, instead of sharing their generic approach. The answer is almost always along the lines of there’s a document, or a ticketing system where this is written down. However, when we find and start reading this document, it’s usually a disappointment and does a poor job in conveying takeaways.</p><p><strong>‘Most incidents are written to be filed, not to be read or learned from. </strong><span>This is what we come across again and again. Teams go through incidents, they file a report, pat themselves in the back, and think they’ve learned from it. In reality, the learning has been a fraction of what it could have been.</span></p><p>‘The current incident handling approaches are only scratching the surface of what we could be doing. In several ways, tech is behind several other industries in how we architect reliable systems.’</p><h2 class="header-anchor-post">What Tech Can Learn From Other Industries<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§what-tech-can-learn-from-other-industries" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.pragmaticengineer.com/i/42747309/what-tech-can-learn-from-other-industries" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><em>What can we as software engineers learn from other industries? </em></p><p>‘Luckily, we have decades of studies on incidents and reliable systems across several industries. And there are plenty of applicable learnings that apply to tech.</p><p><strong>‘A common myth is that distributing learnings from incidents is the biggest blocker on improving more. </strong><span>Many teams and people will believe that if only they find a better way to share incident learnings – like make them easier to search or email them out to a larger group – then this will solve the issue of the organization improving from them.</span></p><p>‘However, this belief has been refuted by research many times. The key challenge is the author of a document cannot predict what will be novel or interesting for the reader. Whoever is writing the incident summary will not be able to tell what information will be well-known to the reader. The person writing the incident summary will also often not write down things they assume everyone else to know. However, many readers will not be familiar with them.</p><p><strong>‘Studies repeatedly show that experts have a hard time describing what makes them an expert. </strong><span>This applies to incidents; experienced engineers who mitigate incidents efficiently will have a hard time describing what it was that allowed them to act as swiftly as they did.</span></p><p>‘Much of how we handle incidents is tacit knowledge, that which is not explicit. The question of “how do we build a better incident handling culture?” is not too different from “how do we help people become experts on a topic?”. And the answer needs to go beyond writing things down. A good example is how, to learn to skate, you cannot just rely on reading books about skating.’</p><p><span>Before I talked with John, I doubted that tech had </span><em>anything</em><span> to learn from other industries. ‘We’re in software, in tech, building things that have never been built before’, I thought. Look at mobile phones, cloud computing or Snapchat; none of this has </span><em>ever</em><span> existed!</span></p><p>However, the more we talked, the more it struck home how outages or incidents; stressful situations when something goes wrong, are not unique to software. In fact, often they don't have anything to do with software. These types of unexpected, disruptive events have been happening since before the invention of the wheel. So of course it makes sense there is an accumulated knowledge on how to prepare people for an incident they are yet to experience.</p><p><strong>My biggest takeaway talking with John was how a written culture is </strong><em><strong>not</strong></em><strong> enough to create a great incident culture. </strong><span>Writing things down is important, but it won’t cut it alone. </span></p><p>I remember the best oncall onboarding process we had at Uber. It had nothing to do with documentation. It was a simulation of an incident. The facilitating team deliberately disrupted a non-critical service similar to an incident that happened in the past. The people being onboarded knew this was not a real incident, but they were called into a Zoom call in which one of them was named as Incident Commander and the facilitators played along. The team then did a debrief and analyzed what they could have done differently. This exercise achieved far more than any studying of documentation could have. </p><p><span>I do not have data to base this on, but it </span><em>felt</em><span> to me like the people who went though this simulation were far more prepared for the real thing. It felt like they had more confidence going on their first oncall, as they had already been through an incident.</span></p><p>This approach got me thinking. ‘Is this why the military does training exercises, despite the high cost?’ John did not answer, but asked the question: ‘Why do you think they do it? Would they do so if reading books or watching videos got them a similar result?’</p><p>There’s a lot that we in the software tech industry can learn about how to build resilient systems, by learning about how other industries have been building resilient systems.</p><h2 class="header-anchor-post">The Opportunity to Build More Resilient Systems<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§the-opportunity-to-build-more-resilient-systems" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.pragmaticengineer.com/i/42747309/the-opportunity-to-build-more-resilient-systems" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>As our conversation closed, John mentioned how he thinks the software industry is, in some ways, ahead of most other industries in building resilient systems. He told me how he invited a person to the Velocity conference, one focused on resilience, performance and security in tech. This person spent decades researching resilient systems in healthcare, and assumed they would teach the audience. Instead, this person was amazed at how much they had to learn from tech. As John put it:</p><p><strong>‘No other industry has as much, nor as detailed incident data available, as tech does. </strong><span>After an outage happens, engineers have access to code and configuration changes, logs and analytics, often down to the millisecond. You have all this data without having to do much preparation, or go through obstacles to get this data.</span></p><p>‘Compare this to, for example, the medical field. There, you have to do huge amounts of preparation to get data on what happens in an operating theatre. You need to install cameras and microphones. You get permission from all parties to record. You operate the recording equipment, then transfer all the data for processing after the operation.</p><p>‘Access to data is far more strict in, for example, the aviation industry. Let’s say you want to investigate a plane near-miss incident. To access the logs, you need to start a special process, and after many approvals, you can talk with the pilot only in the presence of a union representative, and potentially a lawyer. If you forgot to ask something relevant, you need to start the process again.</p><p>‘Most engineers believe progress in the software industry means progress in how we build software. However, I believe the real progress is how we get more and more data to see how incidents unfold to the point that we’ll be able to answer the question: what made this incident difficult?’</p><p><strong>Software organizations already have all the data they need to improve how they operate</strong><span>, we both concluded. This is a massive advantage compared to all other industries.</span></p><p>I was left wondering, do we realize the privileged position tech puts us in, of having both the data to work with, and the autonomy to do so? And for those who do realize, will we take the opportunity to create much more resilient systems, challenging the best practices of 2021, and pushing ahead to a world where we use incidents to learn and adapt, not just track action items?</p><h2 class="header-anchor-post">Great Incident Review Examples<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§great-incident-review-examples" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.pragmaticengineer.com/i/42747309/great-incident-review-examples" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Few things better show the privileged status of tech than how we already have access to some of the best incident reviews ever written. Companies from Cloudflare, through GitLab and many others, have made these available for anyone who wants to read them. Do keep in mind that public blog posts about incidents are not the same as incident analyses intended for an organization to learn from, and will often not represent the whole story.</p><p><a href="https://www.thevoid.community/">The Verica Open Incident database</a><span> built by Courtney Nash is the most exhaustive public incidents database you can find, and one I’d suggest browsing and bookmarking.</span></p><p data-attrs="{&quot;url&quot;:&quot;https://newsletter.pragmaticengineer.com/p/incident-review-best-practices&quot;,&quot;text&quot;:&quot;Keep reading online&quot;,&quot;action&quot;:null,&quot;class&quot;:null}" data-component-name="ButtonCreateButton" class="button-wrapper"><a href="https://newsletter.pragmaticengineer.com/p/incident-review-best-practices" class="button primary"><span>Keep reading online</span></a></p><p><span>Additionally, here are a few incident analyses </span><a href="https://blog.pragmaticengineer.com/postmortem-best-practices#great-incident-review-examples">that I especially enjoyed reading</a><span>.</span></p><h2 class="header-anchor-post">Conclusions<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§conclusions" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.pragmaticengineer.com/i/42747309/conclusions" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>We’ve covered a lot of ground in this issue of the newsletter. Depending on where you are at on your team with incidents, I’ll leave you with one of these pieces of advice:</p><ul><li><p><strong>If you do not yet follow common industry practices</strong><span> like having a well-defined process for raising an incident, doing regular incident reviews, and recording follow-up items, do give this a go. Start step by step, but try to get to a point where responding to incidents is deliberate, not ad hoc.</span></p></li></ul><ul><li><p><strong>If you already have incident review hygiene in place, challenge yourself to make this better.</strong><span> Do you give “soak time” to teams – the time to process what happened – before doing a review? Are your reviews </span><em>truly</em><span> blameless, making people feel good about having gone through an incident, rather than being discreet finger-pointing exercises? Do you have reviews where the goal is not to get to action items, but to discuss learnings?</span></p></li><li><p><strong>Focus on learning over action items</strong><span>, if you want to take incident handling to the next level. As a warning, you should learn to jog before you run. Don’t jump to this step if your team has not gone through all the previous stages. But </span><em>do</em><span> look at companies like Honeycomb or people like John Allspaw as inspiration of how you can focus more on learning, and less on the processes. </span></p></li><li><p><strong>Are you making meaningful changes after incidents?</strong><span> Analyzing why an incident happened is important, but without taking action and making changes, this analysis is not worth much. Often you’ll find the changes you need to make are complex and time-consuming. Are you following through with this work, and making systemic changes to improve the reliability of your systems?</span></p></li></ul><h2 class="header-anchor-post">Additional Resources<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§additional-resources" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.pragmaticengineer.com/i/42747309/additional-resources" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Resources:</p><ul><li><p><a href="https://docs.google.com/spreadsheets/d/1GPINipdf-l2H05iKOUbpkrqwlZ61ZCJDnwY5iE8LtRM/edit?usp=sharing">Aggregated survey results</a><span> on how teams handle incidents.</span></p></li><li><p><span>🔒</span><a href="https://newsletter.pragmaticengineer.com/p/templates-as-inspiration-for-engineering"> Postmortem templates</a><span> with a lightweight and a heavyweight example.</span></p></li></ul><p>Further related reading:</p><ul><li><p><a href="https://www.learningfromincidents.io/">Learning from incidents community</a><span> (LFI) - a community on incidents, software reliability, and the critical role people play in keeping their systems running,</span></p></li><li><p><a href="https://www.wsj.com/articles/the-airline-safety-revolution-11618585543">The airline safety revolution</a><span> from The Wall Street Journal. No commercial airline in the US has had a fatal crash since 2009. The story of how an industry came together and changed, to make this happen.</span></p></li><li><p><a href="https://www.adaptivecapacitylabs.com/blog/2019/11/20/markers-of-progress-incident-analysis/">Markers of progress in incident analysis</a><span> by Adaptive Capacity Labs. How can you tell if an organization is making progress in learning from incidents?</span></p></li></ul><p>Further related videos:</p><ul><li><p><a href="https://confs.space/conf/failover-conf/swim-don-t-sink-why-training-matters-to-a-site-reliability-engineering-practice-jennifer-petoff/">Swim, don’t sink: why training matters for SRE and DevOps practices</a><span> by </span><a href="https://twitter.com/jennski">Jennifer Petoff</a><span>, head of SRE education at Google</span></p></li><li><p><a href="https://www.youtube.com/watch?v=qDfPrl4l31w&amp;t=1s">Food for Thought: What Restaurants Can Teach Us about Reliability</a><span> by </span><a href="https://www.alex-hidalgo.com/">Alex Hidalgo</a><span>, director of SRE at Nobl9</span></p></li><li><p><a href="https://www.youtube.com/watch?v=Sd3DLgRwTyo">Hard problems we handle in incidents but aren’t recognized</a><span> by John Allspaw, founder of Adaptive Capacity Labs</span></p></li></ul><p><em><span>Thanks to </span><a href="https://twitter.com/evnsio">Chris</a><span>, </span><a href="https://twitter.com/allspaw">John</a><span>, </span><a href="https://twitter.com/LauraMDMaguire">Laura</a><span>, </span><a href="https://twitter.com/paulosman">Paul</a><span>, for their input in this article, and </span><a href="https://twitter.com/alexasandei">Alexandru</a><span>, </span><a href="https://twitter.com/johnwards">John</a><span>, </span><a href="https://www.linkedin.com/in/julik/">Julik</a><span>, </span><a href="https://twitter.com/drkurta">Kurt</a><span>, </span><a href="https://www.linkedin.com/in/stadanapisem/">Miljian</a><span>, </span></em><a href="https://www.linkedin.com/in/michal-wegrzyn84/">Michał</a><span> and </span><a href="https://twitter.com/mmelas74">Marco</a><span> for their review comments.</span></p><h4 class="header-anchor-post"><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.pragmaticengineer.com/i/42747309/" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (101)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">101</div></button></div><button tabindex="0" type="button" aria-label="View comments (2)" data-href="https://newsletter.pragmaticengineer.com/p/incident-review-best-practices/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg><div class="label">2</div></button><button tabindex="0" type="button" id="radix-P0-56" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">3</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="visibility-check"></div><div id="discussion" class="pencraft pc-display-flex pc-flexDirection-column pc-gap-16 pc-paddingTop-32 pc-paddingBottom-32 pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-32 pc-reset container"><h4 class="pencraft pc-reset line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Discussion about this post</h4><div class="pencraft pc-alignSelf-flex-start pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Select discussion type" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-70-content-comments" data-state="active" id="radix-P0-70-trigger-comments" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Comments</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-70-content-restacks" data-state="inactive" id="radix-P0-70-trigger-restacks" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Restacks</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div></div></div><div id="substack-comments" class="single-post-section comments-section"><div class="container"><div class="visibility-check"></div><div data-test-id="comment-input" class="pencraft pc-display-flex pc-reset flex-grow-rzmknG"><form class="comment-input-form form-CkZ7Kt"><div style="--scale:32px;" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><div style="--scale:32px;" title="User" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-flexDirection-column pc-boxShadow-xs pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-sm"><textarea name="body" placeholder="Write a comment..." aria-label="Write a comment..." rows="4" class="pencraft input-qHk4bN autogrowing-_ipn9Y textarea-GbEjRX inputText-pV_yWb unstyled-IJLUJU"></textarea></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></form></div><div class="comment-list post-page-root-comment-list"><div class="comment-list-items"><div class="comment"><div id="comment-3366325" class="comment-anchor"></div><div id="comment-3366325-reply" class="comment-anchor"></div><div role="article" aria-label="Comment by Christopher Cartlidge Brown" class="pencraft pc-display-flex pc-gap-12 pc-paddingBottom-12 pc-reset comment-content"><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><a href="https://substack.com/profile/49909944-christopher-cartlidge-brown?utm_source=comment" aria-label="View Christopher Cartlidge Brown's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:32px;" tabindex="0" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6"><div style="--scale:32px;" title="Christopher Cartlidge Brown" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!yhxI!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fb0f0bd21-e89d-4c0f-b544-a0dbe94373c2_144x144.png 1x, https://substackcdn.com/image/fetch/$s_!yhxI!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fb0f0bd21-e89d-4c0f-b544-a0dbe94373c2_144x144.png 2x, https://substackcdn.com/image/fetch/$s_!yhxI!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fb0f0bd21-e89d-4c0f-b544-a0dbe94373c2_144x144.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!yhxI!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fb0f0bd21-e89d-4c0f-b544-a0dbe94373c2_144x144.png" alt="Christopher Cartlidge Brown's avatar" srcset="https://substackcdn.com/image/fetch/$s_!yhxI!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fb0f0bd21-e89d-4c0f-b544-a0dbe94373c2_144x144.png 1x, https://substackcdn.com/image/fetch/$s_!yhxI!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fb0f0bd21-e89d-4c0f-b544-a0dbe94373c2_144x144.png 2x, https://substackcdn.com/image/fetch/$s_!yhxI!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fb0f0bd21-e89d-4c0f-b544-a0dbe94373c2_144x144.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-4 pc-reset"><div class="pencraft pc-display-flex pc-minWidth-0 pc-gap-8 pc-alignItems-center pc-justifyContent-space-between pc-reset line-height-20-t4M0El font-text-qe4AeH size-15-Psle70 weight-regular-mUq6Gb"><div class="pencraft pc-display-flex pc-minWidth-0 pc-gap-8 pc-alignItems-center pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-gap-6 pc-reset color-primary-zABazT line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb reset-IxiVJZ comment-author-name"><div class="pencraft pc-display-flex pc-gap-4 pc-minWidth-0 pc-alignItems-center pc-reset"><span class="pencraft pc-reset ellipsis-Jb8om8 line-height-20-t4M0El font-text-qe4AeH size-15-Psle70 weight-medium-fw81nC reset-IxiVJZ"><span style="min-width:0;" data-state="closed"><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a href="https://substack.com/profile/49909944-christopher-cartlidge-brown?utm_source=substack-feed-item" showBack data-native="true" class="link-LIBpto">Christopher Cartlidge Brown</a></span></span></span></div></div><a data-native="true" href="https://newsletter.pragmaticengineer.com/p/incident-review-best-practices/comment/3366325" rel="nofollow" title="Oct 26, 2021, 1:40 PM" class="pencraft pc-reset color-secondary-ls1g8s decoration-hover-underline-ClDVRM reset-IxiVJZ"><span class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb nowrap-QngyoB reset-IxiVJZ">Oct 26, 2021</span></a></div></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><button tabindex="-1" type="button" class="pencraft pc-display-flex pc-gap-4 pc-height-20 pc-paddingLeft-6 pc-paddingRight-6 pc-paddingTop-2 pc-paddingBottom-2 pc-alignItems-center pc-reset font-text-qe4AeH size-11-NuY2Zx weight-medium-fw81nC pencraft tag-XbOVLt theme_accent-Y2sqZY priority_secondary-outline-RpooJS pencraft pc-display-flex pc-height-20 pc-paddingLeft-6 pc-paddingRight-6 pc-paddingTop-2 pc-paddingBottom-2 pc-gap-4 pc-alignItems-center pc-reset cursor-inherit-LxLBJ6 pc-borderRadius-xs font-text-qe4AeH size-11-NuY2Zx weight-medium-fw81nC"><div class="pencraft pc-display-flex pc-alignItems-center pc-reset leading-mI5Ihl fillIcon-dQ0mii"><svg aria-hidden="true" style="height:14px;width:14px;" width="14" height="14" viewBox="0 0 20 20" fill="var(--color-fg-primary)" stroke-width="2.5" stroke="#000" xmlns="http://www.w3.org/2000/svg"><g><path stroke="none" d="M9.99915 16.7256C9.90515 16.7256 9.79102 16.692 9.65674 16.6249C9.52246 16.5622 9.3949 16.4906 9.27405 16.41C8.02974 15.6044 6.94657 14.7584 6.02454 13.8722C5.10697 12.9815 4.3953 12.0662 3.88953 11.1262C3.38375 10.1818 3.13086 9.23067 3.13086 8.27283C3.13086 7.63725 3.23157 7.05762 3.43298 6.53394C3.63888 6.01025 3.92086 5.55819 4.27893 5.17773C4.64148 4.79728 5.05774 4.50635 5.52771 4.30493C6.00216 4.09904 6.51241 3.99609 7.05847 3.99609C7.73433 3.99609 8.31844 4.16618 8.81079 4.50635C9.30762 4.84652 9.70374 5.28963 9.99915 5.83569C10.299 5.28516 10.6951 4.84204 11.1875 4.50635C11.6843 4.16618 12.2707 3.99609 12.9465 3.99609C13.4836 3.99609 13.9894 4.09904 14.4639 4.30493C14.9428 4.50635 15.3613 4.79728 15.7194 5.17773C16.0774 5.55819 16.3572 6.01025 16.5586 6.53394C16.7645 7.05762 16.8674 7.63725 16.8674 8.27283C16.8674 9.23067 16.6145 10.1818 16.1088 11.1262C15.603 12.0662 14.8891 12.9815 13.967 13.8722C13.0495 14.7584 11.9708 15.6044 10.731 16.41C10.6056 16.4906 10.4758 16.5622 10.3416 16.6249C10.2118 16.692 10.0976 16.7256 9.99915 16.7256Z"></path></g></svg></div>Liked by Gergely Orosz</button></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"></div><div class="pencraft pc-display-flex pc-reset comment-menu-trigger-container triggerContainer-eX588u"><button tabindex="0" type="button" aria-label="Ellipsis" id="radix-P0-83" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft trigger-j08Uop iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_quaternary-kpMibu"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-ellipsis"><circle cx="12" cy="12" r="1"></circle><circle cx="19" cy="12" r="1"></circle><circle cx="5" cy="12" r="1"></circle></svg></button></div></div><div class="comment-body expanded"><p><span>This is a great topic and something in my organisation we have thought about a lot on improving. One progress we've introduced that has helped to onboard people into incident handling is to put together a mock incident. We take a previous incident and run through a war game with a set of engineers to try and discover what the problem is/was. We have a manager playing the role of an incident manager in a mock slack room.</span></p><p><span>This has been a good tool for onboarding people onto the process, as usually, the first experience is an issue live in production which can be a daunting task.</span></p></div><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-8 pc-justifyContent-flex-start pc-alignItems-center pc-reset comment-actions withShareButton-hQzuEn"><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a class="pencraft pc-reset link-_X6et2 link-LIBpto"><div class="pencraft pc-display-flex pc-gap-6 pc-alignItems-center pc-reset"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="var(--color-fg-secondary-themed)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA comment-actions-label">Reply</div></div></a></span><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a class="pencraft pc-reset link-_X6et2 link-LIBpto"><div class="pencraft pc-display-flex pc-gap-6 pc-alignItems-center pc-reset"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="var(--color-fg-secondary-themed)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA comment-actions-label">Share</div></div></a></span></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></div></div><div class="comment"><div id="comment-3292674" class="comment-anchor"></div><div id="comment-3292674-reply" class="comment-anchor"></div><div role="article" aria-label="Comment by Elaine May" class="pencraft pc-display-flex pc-gap-12 pc-paddingBottom-12 pc-reset comment-content"><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><a href="https://substack.com/profile/210211-elaine-may?utm_source=comment" aria-label="View Elaine May's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:32px;" tabindex="0" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6"><div style="--scale:32px;" title="Elaine May" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!RIAv!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fccae59ec-99f3-470f-90ff-bb4ba5e564bc_144x144.png 1x, https://substackcdn.com/image/fetch/$s_!RIAv!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fccae59ec-99f3-470f-90ff-bb4ba5e564bc_144x144.png 2x, https://substackcdn.com/image/fetch/$s_!RIAv!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fccae59ec-99f3-470f-90ff-bb4ba5e564bc_144x144.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!RIAv!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fccae59ec-99f3-470f-90ff-bb4ba5e564bc_144x144.png" alt="Elaine May's avatar" srcset="https://substackcdn.com/image/fetch/$s_!RIAv!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fccae59ec-99f3-470f-90ff-bb4ba5e564bc_144x144.png 1x, https://substackcdn.com/image/fetch/$s_!RIAv!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fccae59ec-99f3-470f-90ff-bb4ba5e564bc_144x144.png 2x, https://substackcdn.com/image/fetch/$s_!RIAv!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fccae59ec-99f3-470f-90ff-bb4ba5e564bc_144x144.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-4 pc-reset"><div class="pencraft pc-display-flex pc-minWidth-0 pc-gap-8 pc-alignItems-center pc-justifyContent-space-between pc-reset line-height-20-t4M0El font-text-qe4AeH size-15-Psle70 weight-regular-mUq6Gb"><div class="pencraft pc-display-flex pc-minWidth-0 pc-gap-8 pc-alignItems-center pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-gap-6 pc-reset color-primary-zABazT line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb reset-IxiVJZ comment-author-name"><div class="pencraft pc-display-flex pc-gap-4 pc-minWidth-0 pc-alignItems-center pc-reset"><span class="pencraft pc-reset ellipsis-Jb8om8 line-height-20-t4M0El font-text-qe4AeH size-15-Psle70 weight-medium-fw81nC reset-IxiVJZ"><span style="min-width:0;" data-state="closed"><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a href="https://substack.com/profile/210211-elaine-may?utm_source=substack-feed-item" showBack data-native="true" class="link-LIBpto">Elaine May</a></span></span></span><div class="pencraft pc-display-flex pc-alignItems-center pc-reset flex-auto-j3S2WA"><div data-testid="user-badge" class="pencraft pc-reset cursor-pointer-LYORKw inlineContainer-FbhgTi"><div class="pencraft pc-display-flex pc-reset"><svg width="12" height="12" viewBox="0 0 130 130" fill="none" xmlns="http://www.w3.org/2000/svg" class="subscriberBadge-Lpjxjx"><circle cx="65" cy="65" r="65" fill="var(--color-bg)"></circle><path d="M67.5321 67.2143C62.0162 68.2532 54.1338 68.2193 47.1778 64.9205M62.2892 67.2143C61.2169 61.5215 61.2875 53.3079 64.9106 46.1935M62.2892 62.5387C67.8295 61.4951 75.7573 61.534 82.7358 64.8765M67.6643 62.5635C68.7424 68.2874 68.6652 76.5598 64.9832 83.7008M64.9106 46.1935C66.0867 43.8841 67.6371 41.6906 69.6443 39.7368C77.6175 31.5462 89.5811 30.9614 97.1218 32.3817C98.5421 39.9224 97.9573 51.886 89.7667 59.8592C87.6515 62.032 85.2556 63.6696 82.7358 64.8765M64.9106 46.1935C63.7346 43.8841 62.1841 41.6906 60.177 39.7368C52.2038 31.5462 40.2401 30.9614 32.6995 32.3817C31.2791 39.9224 31.8639 51.886 40.0546 59.8592C42.1955 62.0585 44.6242 63.7095 47.1778 64.9205M82.7358 64.8765C85.2556 66.0834 87.6515 67.7209 89.7667 69.8938C97.9573 77.8669 98.5421 89.8306 97.1218 97.3712C89.5811 98.7916 77.6175 98.2068 69.6443 90.0162C67.6781 88.1022 66.1502 85.9583 64.9832 83.7008M64.9832 83.7008C63.8145 85.9674 62.2826 88.12 60.3092 90.041C52.3361 98.2316 40.3724 98.8164 32.8317 97.3961C31.4114 89.8554 31.9962 77.8918 40.1868 69.9186C42.2908 67.7572 44.6726 66.1255 47.1778 64.9205" stroke="var(--color-fg)" stroke-width="8.125" stroke-linejoin="round"></path></svg></div></div></div></div></div><span style="min-width:0;" data-state="closed"></span><a data-native="true" href="https://newsletter.pragmaticengineer.com/p/incident-review-best-practices/comment/3292674" rel="nofollow" title="Oct 19, 2021, 6:18 PM" class="pencraft pc-reset color-secondary-ls1g8s decoration-hover-underline-ClDVRM reset-IxiVJZ"><span class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb nowrap-QngyoB reset-IxiVJZ">Oct 19, 2021</span></a></div></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><button tabindex="-1" type="button" class="pencraft pc-display-flex pc-gap-4 pc-height-20 pc-paddingLeft-6 pc-paddingRight-6 pc-paddingTop-2 pc-paddingBottom-2 pc-alignItems-center pc-reset font-text-qe4AeH size-11-NuY2Zx weight-medium-fw81nC pencraft tag-XbOVLt theme_accent-Y2sqZY priority_secondary-outline-RpooJS pencraft pc-display-flex pc-height-20 pc-paddingLeft-6 pc-paddingRight-6 pc-paddingTop-2 pc-paddingBottom-2 pc-gap-4 pc-alignItems-center pc-reset cursor-inherit-LxLBJ6 pc-borderRadius-xs font-text-qe4AeH size-11-NuY2Zx weight-medium-fw81nC"><div class="pencraft pc-display-flex pc-alignItems-center pc-reset leading-mI5Ihl fillIcon-dQ0mii"><svg aria-hidden="true" style="height:14px;width:14px;" width="14" height="14" viewBox="0 0 20 20" fill="var(--color-fg-primary)" stroke-width="2.5" stroke="#000" xmlns="http://www.w3.org/2000/svg"><g><path stroke="none" d="M9.99915 16.7256C9.90515 16.7256 9.79102 16.692 9.65674 16.6249C9.52246 16.5622 9.3949 16.4906 9.27405 16.41C8.02974 15.6044 6.94657 14.7584 6.02454 13.8722C5.10697 12.9815 4.3953 12.0662 3.88953 11.1262C3.38375 10.1818 3.13086 9.23067 3.13086 8.27283C3.13086 7.63725 3.23157 7.05762 3.43298 6.53394C3.63888 6.01025 3.92086 5.55819 4.27893 5.17773C4.64148 4.79728 5.05774 4.50635 5.52771 4.30493C6.00216 4.09904 6.51241 3.99609 7.05847 3.99609C7.73433 3.99609 8.31844 4.16618 8.81079 4.50635C9.30762 4.84652 9.70374 5.28963 9.99915 5.83569C10.299 5.28516 10.6951 4.84204 11.1875 4.50635C11.6843 4.16618 12.2707 3.99609 12.9465 3.99609C13.4836 3.99609 13.9894 4.09904 14.4639 4.30493C14.9428 4.50635 15.3613 4.79728 15.7194 5.17773C16.0774 5.55819 16.3572 6.01025 16.5586 6.53394C16.7645 7.05762 16.8674 7.63725 16.8674 8.27283C16.8674 9.23067 16.6145 10.1818 16.1088 11.1262C15.603 12.0662 14.8891 12.9815 13.967 13.8722C13.0495 14.7584 11.9708 15.6044 10.731 16.41C10.6056 16.4906 10.4758 16.5622 10.3416 16.6249C10.2118 16.692 10.0976 16.7256 9.99915 16.7256Z"></path></g></svg></div>Liked by Gergely Orosz</button></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"></div><div class="pencraft pc-display-flex pc-reset comment-menu-trigger-container triggerContainer-eX588u"><button tabindex="0" type="button" aria-label="Ellipsis" id="radix-P0-93" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft trigger-j08Uop iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_quaternary-kpMibu"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-ellipsis"><circle cx="12" cy="12" r="1"></circle><circle cx="19" cy="12" r="1"></circle><circle cx="5" cy="12" r="1"></circle></svg></button></div></div><div class="comment-body expanded"><p><span>Thank you for compiling all this information. I learned a lot about incident management in my last job and am working to improve incident management in my current job, so this is especially relevant to me in my current role.</span></p><p><span>One thing I didn't see mentioned is analyzing groups of incidents for commonalities and trends. For instance, one analysis I did showed that a large percentage of our high severity incidents were due to problems with scalability. Because we didn't understand the drivers of scalability, we would run out of a given resource and that would cause an incident. After each incident, we had a post-mortem, but we didn't observe and address the overall trend.</span></p><p><span>Also, Commonplace (a newsletter run by a guy named Cedric) has several articles on extracting tacit knowledge. Sources of Power by Gary Klein also has insights regarding how people develop and use tacit knowledge.</span></p></div><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-8 pc-justifyContent-flex-start pc-alignItems-center pc-reset comment-actions withShareButton-hQzuEn"><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a class="pencraft pc-reset link-_X6et2 link-LIBpto"><div class="pencraft pc-display-flex pc-gap-6 pc-alignItems-center pc-reset"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="var(--color-fg-secondary-themed)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA comment-actions-label">Reply</div></div></a></span><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a class="pencraft pc-reset link-_X6et2 link-LIBpto"><div class="pencraft pc-display-flex pc-gap-6 pc-alignItems-center pc-reset"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="var(--color-fg-secondary-themed)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA comment-actions-label">Share</div></div></a></span></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></div></div></div></div></div></div></div><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-111-content-top" data-state="active" id="radix-P0-111-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-111-content-new" data-state="inactive" id="radix-P0-111-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-111-content-community" data-state="inactive" id="radix-P0-111-trigger-community" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Discussions</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer themed-background"><div class="container"><div class="footer-terms"><span>© 2026 Gergely Orosz</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&amp;utm_medium=web&amp;utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&amp;utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="prTDrEptRhTGJ_GBwV-4sg">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
</div>
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
<script nonce="prTDrEptRhTGJ_GBwV-4sg">
window.Sentry && window.Sentry.onLoad(function() {
window.Sentry.init({
environment: window._preloads.sentry_environment,
dsn: window._preloads.sentry_dsn,
})
})
</script>
<script nonce="prTDrEptRhTGJ_GBwV-4sg">window._preloads = JSON.parse("{\"cspNonce\":\"prTDrEptRhTGJ_GBwV-4sg\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"default\"},\"base_url\":\"https://newsletter.pragmaticengineer.com\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":30107029,\"byline_images_enabled\":true,\"bylines_enabled\":true,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Gergely Orosz\",\"cover_photo_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/5a9e5545-0407-4cd2-9fd1-cc2c3882fe59_1279x268.png\",\"created_at\":\"2021-08-25T13:08:12.798Z\",\"custom_domain_optional\":false,\"custom_domain\":\"newsletter.pragmaticengineer.com\",\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":\"ce270f3a\",\"default_show_guest_bios\":true,\"email_banner_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/3f6c431d-310a-45a0-90fe-3dc43354ef38_1100x220.png\",\"email_from_name\":\"The Pragmatic Engineer\",\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":[],\"free_subscription_benefits\":[\"Shorter articles every Tuesday - with a full article once a month\"],\"ga_pixel_id\":null,\"google_site_verification_token\":null,\"google_tag_manager_token\":null,\"hero_image\":null,\"hero_text\":\"Big Tech and startups, from the inside. Highly relevant for software engineers, AI engineers and engineering leaders, useful for those working in tech.\",\"hide_intro_subtitle\":null,\"hide_intro_title\":null,\"hide_podcast_feed_link\":false,\"homepage_type\":\"newspaper\",\"id\":458709,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":null,\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!eod5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png\",\"minimum_group_size\":3,\"moderation_enabled\":true,\"name\":\"The Pragmatic Engineer\",\"paid_subscription_benefits\":[\"Full articles every Tuesday and Thursday\",\"Access to resources and templates for engineering managers and engineers\",\"Access to the complete archive, see all comments and comment on articles\"],\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"enabled\",\"paywall_free_trial_enabled\":false,\"podcast_art_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/1cfbfe46-530d-4518-85e6-bef3901747b1_3000x3000.png\",\"paid_podcast_episode_art_url\":null,\"podcast_byline\":\"Gergely Orosz\",\"podcast_description\":\"Software engineering at Big Tech and startups, from the inside. Deepdives with experienced engineers and tech professionals who share their hard-earned lessons, interesting stories and advice they have on building software. \\n\\nEspecially relevant for software engineers and engineering leaders: useful for those working in tech.\",\"podcast_enabled\":true,\"podcast_feed_url\":null,\"podcast_title\":\"The Pragmatic Engineer\",\"post_preview_limit\":null,\"primary_user_id\":30107029,\"require_clickthrough\":false,\"show_pub_podcast_tab\":true,\"show_recs_on_homepage\":false,\"subdomain\":\"pragmaticengineer\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#FF6B00\",\"theme_var_color_links\":false,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":null,\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":[{\"id\":\"monthly15usd\",\"object\":\"plan\",\"active\":true,\"aggregate_usage\":null,\"amount\":1500,\"amount_decimal\":\"1500\",\"billing_scheme\":\"per_unit\",\"created\":1630008931,\"currency\":\"usd\",\"interval\":\"month\",\"interval_count\":1,\"livemode\":true,\"metadata\":{\"substack\":\"yes\"},\"meter\":null,\"nickname\":\"$15 a month\",\"product\":\"prod_K735TEUdeGQPmt\",\"tiers_mode\":null,\"transform_usage\":null,\"trial_period_days\":null,\"usage_type\":\"licensed\",\"currency_options\":{\"aud\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":2200,\"unit_amount_decimal\":\"2200\"},\"brl\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":8000,\"unit_amount_decimal\":\"8000\"},\"cad\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":2200,\"unit_amount_decimal\":\"2200\"},\"chf\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":1300,\"unit_amount_decimal\":\"1300\"},\"dkk\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":10000,\"unit_amount_decimal\":\"10000\"},\"eur\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":1400,\"unit_amount_decimal\":\"1400\"},\"gbp\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":1200,\"unit_amount_decimal\":\"1200\"},\"inr\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":144000,\"unit_amount_decimal\":\"144000\"},\"jpy\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":2500,\"unit_amount_decimal\":\"2500\"},\"mxn\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":26500,\"unit_amount_decimal\":\"26500\"},\"nok\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":14500,\"unit_amount_decimal\":\"14500\"},\"nzd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":2600,\"unit_amount_decimal\":\"2600\"},\"pln\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":6000,\"unit_amount_decimal\":\"6000\"},\"sek\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":14500,\"unit_amount_decimal\":\"14500\"},\"usd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":1500,\"unit_amount_decimal\":\"1500\"}}},{\"id\":\"yearly150usd\",\"object\":\"plan\",\"active\":true,\"aggregate_usage\":null,\"amount\":15000,\"amount_decimal\":\"15000\",\"billing_scheme\":\"per_unit\",\"created\":1633977225,\"currency\":\"usd\",\"interval\":\"year\",\"interval_count\":1,\"livemode\":true,\"metadata\":{\"substack\":\"yes\"},\"meter\":null,\"nickname\":\"$150 a year\",\"product\":\"prod_KOFpMRj8emeVAU\",\"tiers_mode\":null,\"transform_usage\":null,\"trial_period_days\":null,\"usage_type\":\"licensed\",\"currency_options\":{\"aud\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":22000,\"unit_amount_decimal\":\"22000\"},\"brl\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":77000,\"unit_amount_decimal\":\"77000\"},\"cad\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":21500,\"unit_amount_decimal\":\"21500\"},\"chf\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":12500,\"unit_amount_decimal\":\"12500\"},\"dkk\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":98000,\"unit_amount_decimal\":\"98000\"},\"eur\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":13500,\"unit_amount_decimal\":\"13500\"},\"gbp\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":11500,\"unit_amount_decimal\":\"11500\"},\"inr\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":1436000,\"unit_amount_decimal\":\"1436000\"},\"jpy\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":25000,\"unit_amount_decimal\":\"25000\"},\"mxn\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":262500,\"unit_amount_decimal\":\"262500\"},\"nok\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":144500,\"unit_amount_decimal\":\"144500\"},\"nzd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":26000,\"unit_amount_decimal\":\"26000\"},\"pln\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":57000,\"unit_amount_decimal\":\"57000\"},\"sek\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":145000,\"unit_amount_decimal\":\"145000\"},\"usd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"inclusive\",\"unit_amount\":15000,\"unit_amount_decimal\":\"15000\"}}}],\"stripe_user_id\":\"acct_1JSopsA7n83QB5wA\",\"stripe_country\":\"NL\",\"stripe_publishable_key\":\"pk_live_51JSopsA7n83QB5wAw9IUzfjqUCZGQg4SiKPHVTvZ9kYb7CCt6Ef1740RiT4hrkCLTlyQiFY1xYkVzJwauwGjRw0L00fCfhzbwF\",\"stripe_platform_account\":\"US\",\"automatic_tax_enabled\":true,\"author_name\":\"Gergely Orosz\",\"author_handle\":\"pragmaticengineer\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!CPFa!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png\",\"author_bio\":\"Big Tech and startups from the inside. Especially relevant for software engineers / AI engineers, useful for anyone working in tech.\",\"twitter_screen_name\":\"GergelyOrosz\",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":null,\"web_bg_color\":\"#ffffff\",\"cover_bg_color\":null,\"publication_id\":458709,\"color_links\":null,\"font_preset_heading\":\"fancy_serif\",\"font_preset_body\":\"sans\",\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"feature-media\",\"home_posts\":\"list\",\"home_show_top_posts\":false,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":null,\"custom_footer\":null,\"social_media_links\":null,\"font_options\":null,\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":null,\"default_group_coupon_percent_off\":\"10.00\",\"default_group_coupon_include_founding\":false,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2021-08-30T11:24:02.082Z\",\"has_podcast\":true,\"has_free_podcast\":true,\"has_subscriber_only_podcast\":false,\"has_community_content\":true,\"rankingDetail\":\"Tens of thousands of paid subscribers\",\"rankingDetailFreeIncluded\":\"Millions of subscribers\",\"rankingDetailOrderOfMagnitude\":10000,\"rankingDetailFreeIncludedOrderOfMagnitude\":1000000,\"rankingDetailFreeSubscriberCount\":\"Over 1,100,000 subscribers\",\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u0639\u0634\u0631\u0627\u062A \u0627\u0644\u0622\u0644\u0627\u0641 \u0645\u0646 \u0627\u0644\u0645\u0634\u062A\u0631\u0643\u064A\u0646 \u0627\u0644\u0645\u062F\u0641\u0648\u0639\u064A\u0646\"},\"ca\":{\"rankingDetail\":\"Desenes de milers de subscriptors de pagament\"},\"da\":{\"rankingDetail\":\"Titusindvis af betalte abonnenter\"},\"de\":{\"rankingDetail\":\"Zehntausende von Paid-Abonnenten\"},\"es\":{\"rankingDetail\":\"Decenas de miles de suscriptores de pago\"},\"fr\":{\"rankingDetail\":\"Des dizaines de milliers d'abonn\u00E9s payants\"},\"id\":{\"rankingDetail\":\"Puluhan ribu pelanggan berbayar\"},\"ja\":{\"rankingDetail\":\"\u4F55\u4E07\u4EBA\u3082\u306E\u6709\u6599\u767B\u9332\u8005\"},\"ko\":{\"rankingDetail\":\"\uC218\uB9CC \uBA85\uC758 \uC720\uB8CC \uAD6C\uB3C5\uC790\"},\"nb\":{\"rankingDetail\":\"Titusenvis av betalende abonnenter\"},\"nl\":{\"rankingDetail\":\"Tienduizenden betalende abonnees\"},\"pl\":{\"rankingDetail\":\"Dziesi\u0105tki tysi\u0119cy p\u0142ac\u0105cych subskrybent\u00F3w\"},\"pt\":{\"rankingDetail\":\"Dezenas de milhares de subscritores pagos\"},\"pt-br\":{\"rankingDetail\":\"Dezenas de milhares de assinantes pagantes\"},\"en-gb\":{\"rankingDetail\":\"Tens of thousands of paid subscribers\"},\"it\":{\"rankingDetail\":\"Decine di migliaia di abbonati a pagamento\"},\"tr\":{\"rankingDetail\":\"On binlerce \u00FCcretli abone\"},\"sv\":{\"rankingDetail\":\"Tiotusentals betalande prenumeranter\"},\"fi\":{\"rankingDetail\":\"Tens of thousands of paid subscribers\"},\"is\":{\"rankingDetail\":\"Tens of thousands of paid subscribers\"},\"zh-hans\":{\"rankingDetail\":\"\u6570\u4E07\u540D\u4ED8\u8D39\u8BA2\u9605\u8005\"},\"zh-hant\":{\"rankingDetail\":\"\u6578\u842C\u540D\u4ED8\u8CBB\u8A02\u95B1\u8005\"},\"en\":{\"rankingDetail\":\"Tens of thousands of paid subscribers\"}},\"freeSubscriberCount\":\"1,100,000\",\"freeSubscriberCountOrderOfMagnitude\":\"1.1M+\",\"author_bestseller_tier\":10000,\"author_badge\":{\"type\":\"bestseller\",\"tier\":10000},\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":false,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":null,\"primary_profile_name\":\"Gergely Orosz\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!CPFa!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png\",\"no_follow\":false,\"sponsorshipCampaigns\":{\"0829178b-acbc-41b2-a794-66111c122926\":{\"id\":\"0829178b-acbc-41b2-a794-66111c122926\",\"publication_id\":458709,\"title\":\"Main Podcast Sponsorship\",\"short_text\":\"PRESENTED BY\",\"image_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/fdfb2f9c-ed69-4355-af86-3d711ff5b266_2754x396.png\",\"image_alt_text\":\"Antithesis\",\"link_url\":\"https://turbopuffer.com/pragmatic\",\"exclude_audience_minimum\":null,\"created_at\":\"2025-08-27T17:59:52.566Z\",\"updated_at\":\"2025-08-27T17:59:52.566Z\",\"campaign_type\":\"podcast\",\"doc_content\":null,\"status\":\"active\",\"episode_short_text\":\"THIS EPISODE IS PRESENTED BY\",\"section_id\":null,\"description\":null,\"campaign_source\":\"third_party\",\"hidden\":false,\"sponsor_id\":null,\"sponsor_campaign_id\":null}},\"paywall_chat\":\"free\",\"sections\":[{\"id\":24961,\"created_at\":\"2022-08-18T13:19:56.809Z\",\"updated_at\":\"2024-06-18T17:22:31.339Z\",\"publication_id\":458709,\"name\":\"The Pulse\",\"description\":\"Keeping a pulse on the software engineering industry, every Thursday. Insights, patterns and trends within Big Tech and startups. Exclusive analysis of what is happening, and why.\",\"slug\":\"the-pulse\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":1,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"unifiedPodcastSettings\":null,\"showLinks\":[],\"pageTheme\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":132248,\"created_at\":\"2024-06-18T15:49:33.192Z\",\"updated_at\":\"2024-06-18T16:23:15.198Z\",\"publication_id\":458709,\"name\":\"Deepdives\",\"description\":\"Long-form educational articles every Tuesday. These articles typically go deep into a topic relevant for software engineers or engineering managers, and tend to offer actionable advice.\",\"slug\":\"deepdives\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":2,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"unifiedPodcastSettings\":null,\"showLinks\":[],\"pageTheme\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":182752,\"created_at\":\"2025-01-27T17:55:26.493Z\",\"updated_at\":\"2025-01-27T21:26:37.087Z\",\"publication_id\":458709,\"name\":\"Announcements\",\"description\":\"Issues that are neither deepdives, nor The Pulse issues\",\"slug\":\"announcements\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":3,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"unifiedPodcastSettings\":null,\"showLinks\":[],\"pageTheme\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}],\"podcastTabInfo\":{\"hasMultiplePodcasts\":false,\"firstPodcastSectionId\":null},\"multipub_migration\":null,\"navigationBarItems\":[{\"id\":\"bc2883cd-a436-40ba-ad79-bb9944b34f1b\",\"publication_id\":458709,\"sibling_rank\":0,\"link_title\":null,\"link_url\":null,\"section_id\":24961,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"postTag\":null,\"section\":{\"id\":24961,\"slug\":\"the-pulse\",\"name\":\"The Pulse\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":null,\"newsletter_enabled\":null},\"children\":[]},{\"id\":\"13219a85-602a-4315-a13a-e8cf8114b672\",\"publication_id\":458709,\"sibling_rank\":9999,\"link_title\":null,\"link_url\":null,\"section_id\":132248,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"postTag\":null,\"section\":{\"id\":132248,\"slug\":\"deepdives\",\"name\":\"Deepdives\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":null,\"newsletter_enabled\":null},\"children\":[]},{\"id\":\"91c2c7ff-d4f9-4f8a-8ca2-c269ddb299e7\",\"publication_id\":458709,\"sibling_rank\":9999,\"link_title\":null,\"link_url\":null,\"section_id\":182752,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"postTag\":null,\"section\":{\"id\":182752,\"slug\":\"announcements\",\"name\":\"Announcements\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":null,\"newsletter_enabled\":null},\"children\":[]}],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Gergely Orosz\",\"handle\":\"pragmaticengineer\",\"role\":\"admin\",\"owner\":true,\"user_id\":30107029,\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png\",\"bio\":\"Big Tech and startups from the inside. Especially relevant for software engineers / AI engineers, useful for anyone working in tech.\",\"status\":{\"bestsellerTier\":10000,\"subscriberTier\":1,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"bestseller\",\"tier\":10000},\"subscriber\":null}},{\"name\":\"Pen Name Consulting\",\"handle\":\"pennameconsulting\",\"role\":\"contributor\",\"owner\":false,\"user_id\":98424238,\"photo_url\":\"https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F396242ba-4052-49f1-9591-aa0cb88022ea_144x144.png\",\"bio\":null,\"status\":{\"bestsellerTier\":10000,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"bestseller\",\"tier\":10000},\"subscriber\":null}}],\"threads_v2_enabled\":false,\"viralGiftsConfig\":{\"id\":\"4f27054e-d0f8-4806-ac93-003d99963c07\",\"publication_id\":458709,\"enabled\":true,\"gifts_per_user\":5,\"gift_length_months\":1,\"send_extra_gifts\":true,\"message\":\"Big Tech and high-growth startups, from the inside. Highly relevant for software engineers and managers, useful for those working in tech.\",\"created_at\":\"2022-10-06T13:48:22.068838+00:00\",\"updated_at\":\"2022-10-06T13:48:22.068838+00:00\",\"days_til_invite\":14,\"send_emails\":true,\"show_link\":null},\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"iap_advanced_plans\":[{\"sku\":\"UBklw85ezDfkuQDgTb\",\"publication_id\":\"458709\",\"is_active\":true,\"price_base_units\":2100,\"currency_alpha3\":\"usd\",\"period\":\"month\",\"created_at\":\"2025-08-18T13:43:25.818Z\",\"updated_at\":\"2025-08-18T13:43:25.818Z\",\"id\":\"169824\",\"payout_amount_base_units\":150,\"alternate_currencies\":{\"aud\":3300,\"brl\":11500,\"cad\":3000,\"chf\":1700,\"dkk\":13500,\"eur\":1800,\"gbp\":1600,\"mxn\":39500,\"nok\":21500,\"nzd\":3600,\"pln\":8000,\"sek\":20500},\"is_founding\":false,\"display_name\":\"The Pragmatic Engineer (Monthly)\",\"display_price\":\"$21\"},{\"sku\":\"wq04xWQTHQQDfsM8oV\",\"publication_id\":\"458709\",\"is_active\":true,\"price_base_units\":20500,\"currency_alpha3\":\"usd\",\"period\":\"year\",\"created_at\":\"2025-08-18T13:43:25.835Z\",\"updated_at\":\"2025-08-18T13:43:25.835Z\",\"id\":\"169825\",\"payout_amount_base_units\":1500,\"alternate_currencies\":{\"aud\":31500,\"brl\":111000,\"cad\":28500,\"chf\":17000,\"dkk\":131000,\"eur\":17500,\"gbp\":15500,\"mxn\":384000,\"nok\":208500,\"nzd\":35000,\"pln\":75000,\"sek\":196000},\"is_founding\":false,\"display_name\":\"The Pragmatic Engineer (Yearly)\",\"display_price\":\"$205\"}],\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://newsletter.pragmaticengineer.com\",\"hostname\":\"newsletter.pragmaticengineer.com\",\"is_on_substack\":false,\"show_links\":[{\"id\":23747,\"publication_id\":458709,\"section_id\":null,\"url\":\"https://podcasts.apple.com/us/podcast/the-pragmatic-engineer-podcast/id1769051199\",\"platform\":\"apple_podcasts\"},{\"id\":23797,\"publication_id\":458709,\"section_id\":null,\"url\":\"https://open.spotify.com/show/2Bho9xCbOQMWMJ7UKmqCzD\",\"platform\":\"spotify\"},{\"id\":23799,\"publication_id\":458709,\"section_id\":null,\"url\":\"https://www.youtube.com/@pragmaticengineer\",\"platform\":\"youtube\"},{\"id\":23798,\"publication_id\":458709,\"section_id\":null,\"url\":\"https://open.spotify.com/show/2Bho9xCbOQMWMJ7UKmqCzD\",\"platform\":\"spotify_for_paid_users\"},{\"id\":25077,\"publication_id\":458709,\"section_id\":null,\"url\":\"https://music.youtube.com/playlist?list=PLzwJJv8h-iciW53inSOkQA4mkG8TuQAUh\",\"platform\":\"youtube_music\"}],\"spotify_podcast_settings\":{\"id\":21280,\"publication_id\":458709,\"section_id\":null,\"spotify_uri\":\"spotify:show:2wfSTS6OzM2iOQJWwbL6U0\",\"spotify_podcast_title\":null,\"created_at\":\"2024-09-14T18:44:10.144Z\",\"updated_at\":\"2024-09-17T17:59:03.311Z\",\"currently_published_on_spotify\":false,\"spotify_show_url\":\"https://open.spotify.com/show/2wfSTS6OzM2iOQJWwbL6U0\"},\"unified_podcast_settings\":null,\"podcastPalette\":{\"Vibrant\":{\"rgb\":[244,68,60],\"population\":948},\"DarkVibrant\":{\"rgb\":[125.51941747572819,12.230097087378612,7.0805825242718115],\"population\":0},\"LightVibrant\":{\"rgb\":[244,133,133],\"population\":38},\"Muted\":{\"rgb\":[130,124,132],\"population\":4},\"DarkMuted\":{\"rgb\":[89,89,90],\"population\":133},\"LightMuted\":{\"rgb\":[239,225,225],\"population\":299}},\"pageThemes\":{\"podcast\":{\"id\":13711,\"publication_id\":458709,\"section_id\":null,\"page\":\"podcast\",\"page_hero\":\"podcast\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"left\"}},\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#ff6b00\",\"primary\":{\"r\":255,\"g\":107,\"b\":0,\"a\":1},\"primary_hover\":{\"r\":232,\"g\":88,\"b\":0,\"a\":1},\"primary_elevated\":{\"r\":232,\"g\":88,\"b\":0,\"a\":1},\"secondary\":{\"r\":255,\"g\":107,\"b\":0,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":107,\"b\":0,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":107,\"b\":0,\"a\":0.3},\"dark\":{\"primary\":{\"r\":255,\"g\":107,\"b\":0,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":136,\"b\":66,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":136,\"b\":66,\"a\":1},\"secondary\":{\"r\":255,\"g\":107,\"b\":0,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":107,\"b\":0,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":107,\"b\":0,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":210,\"g\":68,\"b\":0,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":255,\"g\":107,\"b\":0,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":206.90096477355226,\"g\":206.90096477355175,\"b\":206.9009647735519,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#FF6B00\",\"primary\":{\"r\":255,\"g\":107,\"b\":0,\"a\":1},\"primary_hover\":{\"r\":230,\"g\":96,\"b\":0,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":107,\"b\":0,\"a\":1},\"secondary\":{\"r\":255,\"g\":107,\"b\":0,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":255,\"g\":107,\"b\":0,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":136,\"b\":66,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":136,\"b\":66,\"a\":1},\"secondary\":{\"r\":255,\"g\":107,\"b\":0,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":107,\"b\":0,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":107,\"b\":0,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"secondary\":{\"r\":134,\"g\":135,\"b\":135,\"a\":1},\"tertiary\":{\"r\":146,\"g\":146,\"b\":146,\"a\":1},\"accent\":{\"r\":255,\"g\":107,\"b\":0,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":255,\"g\":107,\"b\":0,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"secondary\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"secondary_elevated\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"tertiary\":{\"r\":221,\"g\":221,\"b\":221,\"a\":1},\"quaternary\":{\"r\":183,\"g\":183,\"b\":183,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}},\"wordmark_bg\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"is_dark\":false},\"fonts\":{\"heading\":\"fancy_serif\",\"body\":\"sans\"}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[244,68,68],\"population\":1372},\"DarkVibrant\":{\"rgb\":[125.23333333333333,7.366666666666672,7.366666666666672],\"population\":0},\"LightVibrant\":{\"rgb\":[246,134,134],\"population\":67},\"Muted\":{\"rgb\":[144.5,8.499999999999998,8.499999999999998],\"population\":0},\"DarkMuted\":{\"rgb\":[144.5,8.499999999999998,8.499999999999998],\"population\":0},\"LightMuted\":{\"rgb\":[142.40769230769234,10.59230769230766,10.59230769230766],\"population\":0}},\"section\":{\"id\":132248,\"created_at\":\"2024-06-18T15:49:33.192Z\",\"updated_at\":\"2024-06-18T16:23:15.198Z\",\"publication_id\":458709,\"name\":\"Deepdives\",\"description\":\"Long-form educational articles every Tuesday. These articles typically go deep into a topic relevant for software engineers or engineering managers, and tend to offer actionable advice.\",\"slug\":\"deepdives\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":2,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"unifiedPodcastSettings\":null,\"showLinks\":[],\"pageTheme\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":false,\"customDomain\":\"newsletter.pragmaticengineer.com\"},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":false,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":true,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":10,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":true,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"newbies_only\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"experiment\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":true,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":false,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":true,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":true,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":true,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":true,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":null,\"twitterPixelId\":null,\"parselyPixelId\":null,\"googleTagManagerToken\":null,\"googleAnalytics4Token\":\"G-SL4ERK2LEQ\",\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":true,\"credit_token_enabled\":true,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":false,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":false,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":\"G-SL4ERK2LEQ\",\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"\",\"notes_feed_enabled\":false,\"paywall_unlock_tokens\":false,\"post_preview_crop_gravity\":\"center\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":false,\"reader_referrals_leaderboard_enabled\":false,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":false,\"local_currency_modal_seen\":true,\"local_payment_methods_modal_seen\":true,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":true,\"welcome_page_opt_out_text\":\"No thanks\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":true,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://newsletter.pragmaticengineer.com/p/incident-review-best-practices\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":42747309,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2021-10-19T15:14:03.962Z\",\"updated_at\":\"2025-11-07T15:27:15.404Z\",\"publication_id\":458709,\"search_engine_description\":null,\"search_engine_title\":null,\"section_id\":132248,\"should_send_free_preview\":false,\"show_guest_bios\":true,\"slug\":\"incident-review-best-practices\",\"social_title\":\"Incident Review and Postmortem Best Practices\",\"subtitle\":\"A survey of how companies deal with incidents today, and a peek into the best practices of the future.\",\"teaser_post_eligible\":true,\"title\":\"Incident Review and Postmortem Best Practices\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"only_paid\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":3,\"reactions\":{\"\u2764\":101},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":null,\"next_post_slug\":\"engineering-career-paths\",\"cover_image\":\"https://substackcdn.com/image/fetch/$s_!sJy5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png\",\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":{\"post_id\":42747309,\"podcast_episode_number\":null,\"podcast_season_number\":null,\"podcast_episode_type\":null,\"should_syndicate_to_other_feed\":null,\"syndicate_to_section_id\":null,\"hide_from_feed\":false,\"free_podcast_url\":null,\"free_podcast_duration\":null,\"preview_contains_ad\":false,\"was_imported_self_serve_sync\":false,\"draft_free_podcast_url\":null,\"draft_free_podcast_duration\":null},\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"A survey of how companies deal with incidents today, and a peek into the best practices of the future.\",\"body_html\":\"<p><em>\uD83D\uDC4B Hi, this is <a href=\\\"https://twitter.com/gergelyorosz\\\">Gergely</a> with this month\u2019s free edition of the Pragmatic Engineer Newsletter. In every issue, I cover challenges at big tech and high-growth startups through the lens of engineering managers and senior engineers.</em></p><p><em>Subscribe to get weekly issues. Many subscribers expense this newsletter to their learning and development budget.\uD83D\uDC47</em></p><p class=\\\"button-wrapper\\\" data-attrs=\\\"{&quot;url&quot;:&quot;https://newsletter.pragmaticengineer.com/subscribe?&quot;,&quot;text&quot;:&quot;Subscribe now&quot;,&quot;action&quot;:null,&quot;class&quot;:null}\\\" data-component-name=\\\"ButtonCreateButton\\\"><a class=\\\"button primary\\\" href=\\\"https://newsletter.pragmaticengineer.com/subscribe?\\\"><span>Subscribe now</span></a></p><p></p><p>One reason incidents are important is that they often reveal the <em>real</em> state of products, teams or organizations, which is often very different from the imaginary picture that engineering leaders have in their heads. Transparent incident reports and a good incident-handling strategy can inject much-needed realism into the development process. It\u2019s hard to brush aside incidents that have caused specific damage, indeed these incidents are often powerful ways to make cases for work that would otherwise be delayed indefinitely.</p><p>The only certain thing about outages is that they will always happen. Everything else is up to us. How much effort do we put into preventing them, or into mitigating them quickly, or learning from them?</p><p>I\u2019ve talked with dozens of engineers at a variety of companies and have found that no one is fully happy with how they are handling incidents.&nbsp;</p><p>In this issue we cover:</p><ul><li><p><strong>Common incident handling practices across the industry. </strong>What are the trends on how tech companies approach incidents today?</p></li><li><p><strong>Incident review best practices</strong>. What are processes, tools and approaches that we can point to as sensible practices?</p></li><li><p><strong>Incident review practices of tomorrow</strong>. A few teams and companies have moved beyond what we\u2019d call the best practices of today. What is their approach and how is it working?</p></li><li><p><strong>What tech can learn from incident handling in other industries.</strong> Incidents are not unique to tech; fields like healthcare, the military and many others have a long history of efficiently dealing with incidents. What can we learn from them?</p></li><li><p><strong>Incident review/postmortem examples and templates.</strong> A selection of case studies you can get inspiration from and \uD83D\uDD12<a href=\\\"https://newsletter.pragmaticengineer.com/p/templates-as-inspiration-for-engineering\\\">subscriber-only postmortem templates</a>.</p></li></ul><p>The oncall process \u2013 monitoring and alerting \u2013 is beyond the scope of this article. We cover what happens starting from when an outage or incident is confirmed.</p><p><em>Note: This article mentions vendors specializing in incident handling-related services. I have not been paid to mention these companies. My newsletter is fully independent, does not offer sponsorships and I have no commercial affiliation with the vendors mentioned.</em></p><h2>How Companies Handle Incidents</h2><p>More than 60 teams have shared information on how they respond to outages, and what happens afterwards. <strong><a href=\\\"https://docs.google.com/spreadsheets/d/1GPINipdf-l2H05iKOUbpkrqwlZ61ZCJDnwY5iE8LtRM/edit?usp=sharing\\\">See aggregated survey results here.</a></strong></p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!sJy5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!sJy5!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 424w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 848w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 1272w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!sJy5!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png\\\" width=\\\"1456\\\" height=\\\"690\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:690,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:1769147,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!sJy5!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 424w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 848w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 1272w, https://substackcdn.com/image/fetch/$s_!sJy5!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9ccc5d8c-d40e-4fcb-930b-57c49b7275b8_1740x824.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a><figcaption class=\\\"image-caption\\\">How teams respond to outages - some of the results.</figcaption></figure></div><p>98.5% of companies sharing details have an incident management process in place. The only exception was a Series C property management company which used an ad-hoc process on top of email, to manage incidents. I was expecting more variation on the process, the tooling and the steps, but there were large overlaps across companies. This might be explained by selection bias, meaning the survey was mostly filled out by engineers whose teams already take incidents seriously.</p><p>The majority of companies sharing data follow a process that can be summarized like this:</p><ul><li><p><strong>An outage is detected. </strong>This could happen following an automated alert, or someone noticing that something is \u201Coff\u201D.</p></li><li><p><strong>An incident is declared.</strong> An engineer \u2013 often the on-call engineer \u2013 confirms that an outage is happening. They follow an agreed process to declare this incident, which is often a mix of declaring it over chat, and sometimes opening a ticket in a system used to track incidents.</p></li><li><p><strong>The incident is being mitigated.</strong> During mitigation, the engineers involved provide updates. Some companies have more processes around this than others; for example, making it clear who the Incident Leader or Incident Commander is, or expecting regular updates to stakeholders when high-priority outages happen.</p></li><li><p><strong>The incident has been mitigated.</strong> Everyone breathes a sigh of relief.</p></li><li><p><strong>Decompression period.</strong> There is rarely an expectation for engineers to immediately kick off root cause investigation. This is especially true when the mitigation happened outside business hours, say in the middle of the night. However, many companies have a target window to complete analysis within 36-48 hours, which puts pressure on engineers to shorten this decompression time.</p></li><li><p><strong>Incident analysis / postmortem / root cause analysis</strong>. Almost every company calls the investigation that follows by a different name. The idea is the same, the people involved in the incident get together one way or the other, and summarize what happened, and what the learnings are.</p></li><li><p><strong>Incident review</strong>. For outages with a large enough impact, a bigger group reviews the analysis put together in the previous step. Some companies have dedicated incident management teams running this process, some have weekly or bi-weekly meetings with some managers present, while others do it ad hoc.</p></li><li><p><strong>Incident actions tracked for follow up.</strong> In the analysis and review, teams come up with improvements on how to either prevent similar outages, reduce the time to mitigate, or the impact of the outage. During the review, these actions are captured, and added to the roadmap or backlog of the teams owning these.</p></li></ul><p>What tooling do teams use when handling incidents? Based on survey responses, these were the most common mentions:</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!gTi-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!gTi-!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 424w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 848w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 1272w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!gTi-!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png\\\" width=\\\"1254\\\" height=\\\"624\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:624,&quot;width&quot;:1254,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:111147,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!gTi-!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 424w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 848w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 1272w, https://substackcdn.com/image/fetch/$s_!gTi-!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9b1253db-a8b2-4d49-aa54-61ef3978381a_1254x624.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a><figcaption class=\\\"image-caption\\\">Common tooling teams use when handling incidents, based on survey results.</figcaption></figure></div><p><strong>Less common approaches</strong> that a few companies mentioned:</p><ul><li><p><em>Research the same day, postmortem the next day.</em><strong> </strong>\u201COur team does a little research right afterwards, while the incident is still fresh. We then start the postmortem process the next day. It\u2019s down to the engineers to decide how they go about it; the point is they don\u2019t need to start the postmortem right away.\u201D</p></li><li><p><em>Dedicated decompression time.</em> \u201CThe team which owns the service/system that failed takes time out from regular work to decompress (if it was a big outage) and to work on post-incident review and immediate follow-up actions.\u201D</p></li><li><p><em>Create low priority incidents to broadcast risky migrations. Some companies with incident systems that are used to check the health of systems, often create low-priority incidents to raise awareness for risky migrations that could result in temporary issues, but which they are actively monitoring for.</em></p></li></ul><h2>Incident Handling Best Practices</h2><p>What are practices that stand out in their effectiveness, when we think of incidents?&nbsp;</p><ul><li><p><strong>Encourage raising incidents, even when in doubt.</strong> It can be stressful to decide whether or not an incident should be created when something looks \u201Coff\u201D. At the same time, many incidents are either delayed in their response, or are missed completely because someone did not ring the alarm when they had a nagging suspicion. Create a culture where it\u2019s encouraged to raise an incident. Even if it turns out to not be an outage, it\u2019s worthwhile learning to investigate <em>why</em> it felt like something was wrong.</p></li><li><p><strong>Be clear on roles during incidents</strong>. Make it clear who is in charge of mitigating the incident, and who should communicate with stakeholders. Many places call this role the Incident Commander. The role of leading mitigation and communicating might be the same. However, for high severity outages, or when many stakeholders need to be kept in on how it\u2019s going, it might make sense to separate them.</p></li><li><p><strong>Define incident severity levels ahead of time</strong>. Have a shared understanding of what makes an outage more \u2018important\u2019. Write down the definition, and use the same definitions across the organization. Most companies define severity levels based on how customers might be impacted, and the impact radius based on what percentage of these customers are impacted. Over time, you\u2019ll probably need to update your definitions as new types of incidents occur.</p></li><li><p><strong>Have playbooks ready </strong>for common incidents, and make them easy to locate and use. Playbooks could be referenced as part of the alert, or listed at a place where it\u2019s easy for the oncall engineer to find them. Playbooks help with onboarding new members to the rotation, while knowing what to check for \u2013 and in what order \u2013 can help mitigate outages faster.</p></li><li><p><strong>Leave time for engineers to work on an incident review/postmortem</strong>. While it\u2019s always possible to rush this process, you\u2019ll get lower quality reviews if you do so. Instead, give people the space to get together and discuss what happened, and aim for all those involved to have their say.</p></li><li><p><strong>Dig deeper when looking into root causes of the incident.</strong> Don\u2019t stop at the first and often obvious, potential reason. Ask questions to understand how various factors could have contributed to the outage, and what are the things that contributed to those factors.</p></li><li><p><strong>Share incident analysis and learnings with a broader group</strong>, instead of keeping it within the small group which reviews incidents.</p></li><li><p><strong>Find and use tools built for efficient incident handling.</strong> There are companies whose incident review tooling is email-based, and even a simple approach like this can work well enough. However, if you use tools that support the understanding of common incident workflows, you\u2019ll see less confusion, and less tribal knowledge being needed to handle outages. Many companies build their own tooling, and there\u2019s a world of vendors and open source services teams can use. Contributors to this article include engineers from incident handling tools:</p><ul><li><p><a href=\\\"https://incident.io/?utm_source=newsletter&amp;utm_medium=email&amp;utm_campaign=pragmatic_engineer\\\">incident.io</a> (dedicated incident response tools)</p></li><li><p><a href=\\\"https://www.jeli.io/?utm_source=newsletter&amp;utm_medium=email&amp;utm_campaign=pragmatic_engineer\\\">Jeli.io</a> (incident analysis platform)</p></li><li><p><a href=\\\"https://www.blameless.com/?utm_source=newsletter&amp;utm_medium=email&amp;utm_campaign=pragmatic_engineer\\\">Blameless.com</a> (end-to-end platform). </p></li><li><p>Do your research: you\u2019ll find plenty of others as well.</p></li></ul></li></ul><p><strong>Blameless reviews/postmortems </strong>are worth talking more about. When doing a root cause analysis, avoid making it seem like a single person is responsible for the incident. Most outages will be caused by configuration or code changes that someone did, and it\u2019s very easy to find out who it was. However, it could have been <em>anyone</em> else on the team.&nbsp;</p><p>Instead of directly or indirectly putting the blame on one person, go further and look at why systems allowed anyone to make those changes, without providing feedback. If the conditions that allowed the incident to happen are unaddressed, these conditions could trip up someone else on the team in future.</p><p>Some people resist the idea of \u201Cpushing\u201D the use of blameless postmortems across an organization. \u201CWill this not lead to a lack of accountability?\u201D they ask.</p><p>But accountability and a blameless culture are two separate things in my view, which can \u2013 and do \u2013 go hand in hand. Accountability means people take responsibility for their work, and when things inevitably go wrong, they take ownership of fixing things. A blameless approach recognizes and embraces that it\u2019s counterproductive to fault someone for doing something they were unaware would cause an issue. This is especially the case when those people take accountability for addressing the reasons that led to the issue.</p><p><strong>Well-run incident review meetings</strong> are key to sharing context with all involved, and to coming to better learnings. Here is a structure that some of the best run review discussions followed:</p><ul><li><p><em>Do the work upfront. </em>By the time of a meeting, the people involved in the outage should have come together, discussed the incident and their take on it, and put together resources that can be used to give others context.&nbsp;</p></li><li><p><em>Level set in the beginning.</em> Don\u2019t dive in straight to the incident. Start with sharing high-level details and context, so everyone in the room gets on the same page.</p></li><li><p><em>Walk and talk through the timeline as it happened. </em>Don\u2019t just run a script, but talk through what was happening. What looked weird? Why did you have a nagging concern? If you\u2019re in a review, jump in and ask questions. Understanding how things unfolded is much more important than most people realize.</p></li><li><p><em>Keep it blameless.</em> Avoid referring to any one person being the one \u201Cto blame\u201D for the incident.</p></li><li><p><em>Recognize what went well, and what can be improved. Don\u2019t focus only on what\u2019s lacking, but call out actions, processes, systems that worked well, some of which might have been learnings from another incident.</em></p></li><li><p><em>Track actions as they come up. As ideas come up for things that can be changed or improved, keep a running list of these.</em></p></li><li><p><em>Consider separating reviews and action items meetings. By focusing on \\\"what can we learn\\\" in the first review meeting, much better improvement actions emerge in a second meeting focused on action item identification, held a day or more later.</em></p></li><li><p><em>Record the meeting</em> so those who missed it can catch up, and have the full context.</p></li><li><p><em>Open up the meeting to a wider group,</em> so people with additional context, or those curious can join. This practice can help with transparency and spreading of information.</p></li></ul><p>I sat down with <a href=\\\"https://twitter.com/evnsio\\\">Chris Evans</a>, co-founder and CPO at incident.io, formerly head of platform at Monzo. I asked him which other practices he\u2019s seen adopted by engineering teams which are great at incident handling. Here\u2019s what he shared:</p><ul><li><p><strong>Avoid \u201Cblameless buck-passing\u201D. </strong>When a high severity outage happens, there is often pressure from above on the team to \u201Cdo everything so this can never ever happen again.\u201D This approach, in turn, can turn reviews into blameless buck passing exercises, in which those involved try to shift any implied blame away from them or their team, to somewhere else. On the surface, it\u2019s a blameless process. In reality, it\u2019s a poor experience for everyone involved and creates unnecessary pressure.</p></li><li><p><strong>Remove the pressure</strong> of it \u201Cnever happening again\u201D and of all reviews needing to have action items. Do these two things and you\u2019ll have far more effective incident reviews.</p></li><li><p><strong>Give \u201Csoak time\u201D after incidents</strong>. After the outage, leave time for the team to talk and soak up what really happened.</p></li><li><p><strong>Think of incident reviews as a process, not as an artifact.</strong> Too many teams focus on writing the artifact, basically a report, instead of the review being a process that starts before you write anything down, and does not end when a document is complete.</p></li><li><p><strong>Treat an incident as an \u201Caha moment\u201D.</strong> Every incident sheds light on something the team did not know before. What was it?</p></li><li><p><strong>Consider retiring the title \u201Cpostmortem\u201D.</strong> Postmortem means examining a body to determine the cause of death. This has a grim association and starts with the assumption that something terrible has happened.</p></li></ul><h2>Beyond the Best Practices of Today</h2><p>Observability platform <a href=\\\"https://www.honeycomb.io/?utm_source=newsletter&amp;utm_medium=email&amp;utm_campaign=pragmatic_engineer\\\">Honeycomb</a> stood out in their responses to my survey. They were one of the few firms which are moving away from templates and do not track action items. Unlike most other teams which want to do this, they actually took the step.</p><p>From many other teams, I would have interpreted this as trying out something that might or might not work. However, Honeycomb handles huge amounts of data, and provides stricter Service Level Agreements (SLAs) than most of their competitors. They also take reliability very seriously, so much that a 5-minute delay in data processing is <a href=\\\"https://twitter.com/mipsytipsy/status/1443020069483651073?s=20\\\">an outage they publicly report</a>.</p><p>I asked Honeycomb engineer <a href=\\\"https://twitter.com/paulosman\\\">Paul Osman</a> why they made this shift, and how it\u2019s working out. Paul shared how two years ago, they noticed that action items coming up during reviews were not particularly interesting findings; they were mostly things the team was going to do, anyway.</p><p><strong>Focusing on the learning during incident reviews, over explicitly tracking action items,</strong> has been the biggest shift, Paul shared. Teams and individual engineers still create their own tasks, not just for incident reviews, but for everyday work. They\u2019ll also create those tasks even if there\u2019s not an incident review. Paul said:</p><p>\u2018For us, the review is more about \u201Cwho knew what when and how did they know it?\u201D and \u201Chow did our systems surprise us?\u201D instead of \u201Cwhat action items can we extract from this?\u201D</p><p>\u2018Most of the interesting insights we\u2019ve found are in a different category from tasks. They\u2019re things like \u201Cnew traffic patterns can show up as red herrings when debugging\u201D or \u201CHey, this person is needed whenever we have a problem with this system, we should schedule someone to pair with them so they can go on vacation\u201D.&nbsp;</p><p>Honeycomb have left behind some practices that outlived their usefulness, and seem to be better off for it. However, I\u2019d add that they\u2019re still a relatively small group \u2013 around 35 engineers \u2013 at a place where everyone operates with high autonomy.</p><p><strong>Moving on from the Five Whys method</strong> for gathering more information,<strong> </strong>is another approach used by teams which are great at incident handling. The Five Whys is still considered as a best practice by many teams and is a common way to run the root-cause analysis process.<strong> </strong>The idea here is to ask \u201Cwhy\u201D in succession, going deeper and uncovering more information each time.</p><p>The framework is very easy to get started, when teams don\u2019t do much digging into incidents. However, as Andrew Hatch at LinkedIn shares in the talk <a href=\\\"https://www.usenix.org/conference/srecon21/presentation/hatch\\\">Learning More from Complex systems</a>, there are risks to relying on the Five Whys:</p><blockquote><p><em>The danger of the Five Whys is how, by following it, we might miss out on other root causes of the incident. (...) <strong>We\u2019re not broadening our understanding.</strong> We\u2019re just trying to narrow down on one thing, fix it, and hope that this will make the incident not happen again.</em></p></blockquote><p>John Allspaw also advocates against using the Five Whys in his <a href=\\\"https://www.kitchensoap.com/2014/11/14/the-infinite-hows-or-the-dangers-of-the-five-whys/\\\">The Dangers of The Five Whys</a> article. He says that asking \u201Cwhy\u201D is a sure way to start looking for a \u201Cwho\u201D, and to start looking for who is to blame for the outage.</p><p>Instead of asking \u201Cwhy?\u201D multiple times, consider asking \u201Chow did it make sense for someone to do what they did?\u201D. Then dig deeper in gathering the context that made the choices logical, in the context of the people acting. Here are <a href=\\\"http://www.kitchensoap.com/wp-content/uploads/2014/09/Velocity2014-PM-Fac-Handout-Debrief.pdf\\\">some other questions to consider asking</a> to run healthy debriefs.&nbsp;</p><p><strong>Take a socio-technical systems approach to understand the outage.</strong> A root cause analysis or the Five Whys approach can be appealing because they simplify a post-incident effort, but they will not address the deeper systemic issues within your system, says <a href=\\\"https://twitter.com/LauraMDMaguire\\\">Laura Maguire</a>, Head of Research at Jeli.io.&nbsp;</p><p>Socio-technical systems approaches look at both the technical aspects \u2013 what broke \u2013 and the \u2018social\u2019 aspects; how the incident was handled. This gives an oncall team deeper information about software component failures. It also surfaces relevant organizational factors, such as who has specialized knowledge about different aspects of the system and which stakeholders need updates at what frequencies, in order to minimize impacts to their areas of the business. Some guidance for conducting a systemic investigation:</p><ul><li><p><em>Use human error as a starting point for an investigation</em>. For example, if an engineer appears to have made a mistake, look at the processes they used, the training they received, how frequently they interact with that part of the system or codebase, their workload, their oncall schedule and other relevant details that put an error into the broader context. Sidney Dekker, founder of the Safety Science Innovation Lab advocates for this approach in the book, <a href=\\\"https://www.amazon.com/Field-Guide-Understanding-Human-Error/dp/1472439058\\\">The Field Guide to Understanding 'Human Error'</a>.</p></li><li><p><em>Consider organizational factors</em>. Do teams have a headcount that allows them to execute on their roles and responsibilities in a manner that isn\u2019t burning people out? Are there legacy systems that haven\u2019t been dealt with? Is ownership of services clear?&nbsp;</p></li><li><p><em>Gather contextual information relevant to the event</em>. Questions such as \u201CWere you on call?\u201D, \u201CHow did you get brought into the incident\u201D, \u201CHave you worked with this team before?\u201D can reveal important information about the ad hoc or informal ways teams adapt to cope with unusual or difficult outages.&nbsp;</p></li><li><p><em>Encourage participants to recount their experiences as they happened</em>, not as they think they <em>should </em>have responded in hindsight. Focusing on what happened can surface partial or incomplete mental models about how the system works. It can reveal new ways of approaching a problem or monitoring anomalous behaviour and can help a team share specialized knowledge. If psychological safety is present, sharing these stories publicly can allow others to hear what may not be apparent to everyone else.&nbsp;</p></li><li><p><em>A systemic investigation is about asking a different set of questions</em> and recognizing that individual actions and decisions are influenced and constrained by the system of work they take place in.</p></li></ul><h2>Incident Review Practices of Tomorrow</h2><p>I sat down to talk about the future of incident management with <a href=\\\"https://twitter.com/allspaw\\\">John Allspaw</a>, who has been heavily involved in this space for close to a decade. He was engineer #9 at Flickr at the time of Flickr\u2019s massive growth phase. He then became the CTO of Etsy, where he worked for seven years.&nbsp;</p><p>For the past decade, John has been going deeper and deeper into how to build better resilient systems. He enrolled in Lund University in Sweden in the Human Factors and Safety program, a program where some of the leading thinkers in resilience engineering seem to have crossed paths. He has founded <a href=\\\"https://www.adaptivecapacitylabs.com/\\\">Adaptive Capacity Labs</a>, which partners with organizations that want to further improve how they handle and learn from incidents.&nbsp;</p><p>As we talked for an hour, I kept being surprised by the depth of his understanding of resilient systems. It slowly made sense why he spent so much time studying non-tech related subjects, all connected to resilient engineering systems. John also exposed a world of decades-old research outside tech that I would have never thought to look to as inspiration to build better, more reliable systems, but which perhaps we all should. Here\u2019s a summary of our conversation. My questions are in italics.</p><p><em>Why have you been spending so much time on the incident space?</em></p><p><strong>\u2018Incidents create attention energy </strong>around them. When something goes wrong, people pay far more attention to everything around the event than when it\u2019s business as usual. This is also why incidents and outages can be a catalyst to kicking off larger changes within any organization, not just tech companies.\u2019</p><p><em>What is your view on industry best practices which many companies follow, like templates, incident reviews, follow-up items?</em></p><p>\u2018Incident handling practices in the industry are well-intentioned, and point in the right direction. However, these practices are often poorly calibrated. We talk a lot about learning from incidents, and some learning is certainly happening. However, it\u2019s not happening as efficiently as it could.</p><p><strong>\u2018We often confuse fixing things fast with learning. </strong>Take the incident review that most companies follow. There\u2019s an hour, at most, to go through multiple incidents. There is often more focus on generating follow-up items than learning from the incident. In fact, most people seem to think that by generating follow-up items, learning will also happen. However, this is far from the case.</p><p>\u2018When we talk with companies, we ask them to describe how they handle incidents. They jump in, talk about how they detect outages, how they respond, who plays what role, and which tools they use. They\u2019ll often mention cliches like \u201Cwe never let a good incident go to waste.\u201D</p><p>\u2018However, most teams cannot describe a major incident in detail. We typically ask them to talk us through a specific incident, instead of sharing their generic approach. The answer is almost always along the lines of there\u2019s a document, or a ticketing system where this is written down. However, when we find and start reading this document, it\u2019s usually a disappointment and does a poor job in conveying takeaways.</p><p><strong>\u2018Most incidents are written to be filed, not to be read or learned from. </strong>This is what we come across again and again. Teams go through incidents, they file a report, pat themselves in the back, and think they\u2019ve learned from it. In reality, the learning has been a fraction of what it could have been.</p><p>\u2018The current incident handling approaches are only scratching the surface of what we could be doing. In several ways, tech is behind several other industries in how we architect reliable systems.\u2019</p><h2>What Tech Can Learn From Other Industries</h2><p><em>What can we as software engineers learn from other industries?&nbsp;</em></p><p>\u2018Luckily, we have decades of studies on incidents and reliable systems across several industries. And there are plenty of applicable learnings that apply to tech.</p><p><strong>\u2018A common myth is that distributing learnings from incidents is the biggest blocker on improving more. </strong>Many teams and people will believe that if only they find a better way to share incident learnings \u2013 like make them easier to search or email them out to a larger group \u2013 then this will solve the issue of the organization improving from them.</p><p>\u2018However, this belief has been refuted by research many times. The key challenge is the author of a document cannot predict what will be novel or interesting for the reader. Whoever is writing the incident summary will not be able to tell what information will be well-known to the reader. The person writing the incident summary will also often not write down things they assume everyone else to know. However, many readers will not be familiar with them.</p><p><strong>\u2018Studies repeatedly show that experts have a hard time describing what makes them an expert. </strong>This applies to incidents; experienced engineers who mitigate incidents efficiently will have a hard time describing what it was that allowed them to act as swiftly as they did.</p><p>\u2018Much of how we handle incidents is tacit knowledge, that which is not explicit. The question of \u201Chow do we build a better incident handling culture?\u201D is not too different from \u201Chow do we help people become experts on a topic?\u201D. And the answer needs to go beyond writing things down. A good example is how, to learn to skate, you cannot just rely on reading books about skating.\u2019</p><p>Before I talked with John, I doubted that tech had <em>anything</em> to learn from other industries. \u2018We\u2019re in software, in tech, building things that have never been built before\u2019, I thought. Look at mobile phones, cloud computing or Snapchat; none of this has <em>ever</em> existed!</p><p>However, the more we talked, the more it struck home how outages or incidents; stressful situations when something goes wrong, are not unique to software. In fact, often they don't have anything to do with software. These types of unexpected, disruptive events have been happening since before the invention of the wheel. So of course it makes sense there is an accumulated knowledge on how to prepare people for an incident they are yet to experience.</p><p><strong>My biggest takeaway talking with John was how a written culture is </strong><em><strong>not</strong></em><strong> enough to create a great incident culture. </strong>Writing things down is important, but it won\u2019t cut it alone.&nbsp;</p><p>I remember the best oncall onboarding process we had at Uber. It had nothing to do with documentation. It was a simulation of an incident. The facilitating team deliberately disrupted a non-critical service similar to an incident that happened in the past. The people being onboarded knew this was not a real incident, but they were called into a Zoom call in which one of them was named as Incident Commander and the facilitators played along. The team then did a debrief and analyzed what they could have done differently. This exercise achieved far more than any studying of documentation could have.&nbsp;</p><p>I do not have data to base this on, but it <em>felt</em> to me like the people who went though this simulation were far more prepared for the real thing. It felt like they had more confidence going on their first oncall, as they had already been through an incident.</p><p>This approach got me thinking. \u2018Is this why the military does training exercises, despite the high cost?\u2019 John did not answer, but asked the question: \u2018Why do you think they do it? Would they do so if reading books or watching videos got them a similar result?\u2019</p><p>There\u2019s a lot that we in the software tech industry can learn about how to build resilient systems, by learning about how other industries have been building resilient systems.</p><h2>The Opportunity to Build More Resilient Systems</h2><p>As our conversation closed, John mentioned how he thinks the software industry is, in some ways, ahead of most other industries in building resilient systems. He told me how he invited a person to the Velocity conference, one focused on resilience, performance and security in tech. This person spent decades researching resilient systems in healthcare, and assumed they would teach the audience. Instead, this person was amazed at how much they had to learn from tech. As John put it:</p><p><strong>\u2018No other industry has as much, nor as detailed incident data available, as tech does. </strong>After an outage happens, engineers have access to code and configuration changes, logs and analytics, often down to the millisecond. You have all this data without having to do much preparation, or go through obstacles to get this data.</p><p>\u2018Compare this to, for example, the medical field. There, you have to do huge amounts of preparation to get data on what happens in an operating theatre. You need to install cameras and microphones. You get permission from all parties to record. You operate the recording equipment, then transfer all the data for processing after the operation.</p><p>\u2018Access to data is far more strict in, for example, the aviation industry. Let\u2019s say you want to investigate a plane near-miss incident. To access the logs, you need to start a special process, and after many approvals, you can talk with the pilot only in the presence of a union representative, and potentially a lawyer. If you forgot to ask something relevant, you need to start the process again.</p><p>\u2018Most engineers believe progress in the software industry means progress in how we build software. However, I believe the real progress is how we get more and more data to see how incidents unfold to the point that we\u2019ll be able to answer the question: what made this incident difficult?\u2019</p><p><strong>Software organizations already have all the data they need to improve how they operate</strong>, we both concluded. This is a massive advantage compared to all other industries.</p><p>I was left wondering, do we realize the privileged position tech puts us in, of having both the data to work with, and the autonomy to do so? And for those who do realize, will we take the opportunity to create much more resilient systems, challenging the best practices of 2021, and pushing ahead to a world where we use incidents to learn and adapt, not just track action items?</p><h2>Great Incident Review Examples</h2><p>Few things better show the privileged status of tech than how we already have access to some of the best incident reviews ever written. Companies from Cloudflare, through GitLab and many others, have made these available for anyone who wants to read them. Do keep in mind that public blog posts about incidents are not the same as incident analyses intended for an organization to learn from, and will often not represent the whole story.</p><p><a href=\\\"https://www.thevoid.community/\\\">The Verica Open Incident database</a> built by Courtney Nash is the most exhaustive public incidents database you can find, and one I\u2019d suggest browsing and bookmarking.</p><p class=\\\"button-wrapper\\\" data-attrs=\\\"{&quot;url&quot;:&quot;https://newsletter.pragmaticengineer.com/p/incident-review-best-practices&quot;,&quot;text&quot;:&quot;Keep reading online&quot;,&quot;action&quot;:null,&quot;class&quot;:null}\\\" data-component-name=\\\"ButtonCreateButton\\\"><a class=\\\"button primary\\\" href=\\\"https://newsletter.pragmaticengineer.com/p/incident-review-best-practices\\\"><span>Keep reading online</span></a></p><p>Additionally, here are a few incident analyses <a href=\\\"https://blog.pragmaticengineer.com/postmortem-best-practices#great-incident-review-examples\\\">that I especially enjoyed reading</a>.</p><h2>Conclusions</h2><p>We\u2019ve covered a lot of ground in this issue of the newsletter. Depending on where you are at on your team with incidents, I\u2019ll leave you with one of these pieces of advice:</p><ul><li><p><strong>If you do not yet follow common industry practices</strong> like having a well-defined process for raising an incident, doing regular incident reviews, and recording follow-up items, do give this a go. Start step by step, but try to get to a point where responding to incidents is deliberate, not ad hoc.</p></li></ul><ul><li><p><strong>If you already have incident review hygiene in place, challenge yourself to make this better.</strong> Do you give \u201Csoak time\u201D to teams \u2013 the time to process what happened \u2013 before doing a review? Are your reviews <em>truly</em> blameless, making people feel good about having gone through an incident, rather than being discreet finger-pointing exercises? Do you have reviews where the goal is not to get to action items, but to discuss learnings?</p></li><li><p><strong>Focus on learning over action items</strong>, if you want to take incident handling to the next level. As a warning, you should learn to jog before you run. Don\u2019t jump to this step if your team has not gone through all the previous stages. But <em>do</em> look at companies like Honeycomb or people like John Allspaw as inspiration of how you can focus more on learning, and less on the processes.&nbsp;</p></li><li><p><strong>Are you making meaningful changes after incidents?</strong> Analyzing why an incident happened is important, but without taking action and making changes, this analysis is not worth much. Often you\u2019ll find the changes you need to make are complex and time-consuming. Are you following through with this work, and making systemic changes to improve the reliability of your systems?</p></li></ul><h2>Additional Resources</h2><p>Resources:</p><ul><li><p><a href=\\\"https://docs.google.com/spreadsheets/d/1GPINipdf-l2H05iKOUbpkrqwlZ61ZCJDnwY5iE8LtRM/edit?usp=sharing\\\">Aggregated survey results</a> on how teams handle incidents.</p></li><li><p>\uD83D\uDD12<a href=\\\"https://newsletter.pragmaticengineer.com/p/templates-as-inspiration-for-engineering\\\"> Postmortem templates</a> with a lightweight and a heavyweight example.</p></li></ul><p>Further related reading:</p><ul><li><p><a href=\\\"https://www.learningfromincidents.io/\\\">Learning from incidents community</a> (LFI) - a community on incidents, software reliability, and the critical role people play in keeping their systems running,</p></li><li><p><a href=\\\"https://www.wsj.com/articles/the-airline-safety-revolution-11618585543\\\">The airline safety revolution</a> from The Wall Street Journal. No commercial airline in the US has had a fatal crash since 2009. The story of how an industry came together and changed, to make this happen.</p></li><li><p><a href=\\\"https://www.adaptivecapacitylabs.com/blog/2019/11/20/markers-of-progress-incident-analysis/\\\">Markers of progress in incident analysis</a> by Adaptive Capacity Labs. How can you tell if an organization is making progress in learning from incidents?</p></li></ul><p>Further related videos:</p><ul><li><p><a href=\\\"https://confs.space/conf/failover-conf/swim-don-t-sink-why-training-matters-to-a-site-reliability-engineering-practice-jennifer-petoff/\\\">Swim, don\u2019t sink: why training matters for SRE and DevOps practices</a> by <a href=\\\"https://twitter.com/jennski\\\">Jennifer Petoff</a>, head of SRE education at Google</p></li><li><p><a href=\\\"https://www.youtube.com/watch?v=qDfPrl4l31w&amp;t=1s\\\">Food for Thought: What Restaurants Can Teach Us about Reliability</a> by <a href=\\\"https://www.alex-hidalgo.com/\\\">Alex Hidalgo</a>, director of SRE at Nobl9</p></li><li><p><a href=\\\"https://www.youtube.com/watch?v=Sd3DLgRwTyo\\\">Hard problems we handle in incidents but aren\u2019t recognized</a> by John Allspaw, founder of Adaptive Capacity Labs</p></li></ul><p><em>Thanks to <a href=\\\"https://twitter.com/evnsio\\\">Chris</a>, <a href=\\\"https://twitter.com/allspaw\\\">John</a>, <a href=\\\"https://twitter.com/LauraMDMaguire\\\">Laura</a>, <a href=\\\"https://twitter.com/paulosman\\\">Paul</a>, for their input in this article, and <a href=\\\"https://twitter.com/alexasandei\\\">Alexandru</a>, <a href=\\\"https://twitter.com/johnwards\\\">John</a>, <a href=\\\"https://www.linkedin.com/in/julik/\\\">Julik</a>, <a href=\\\"https://twitter.com/drkurta\\\">Kurt</a>, <a href=\\\"https://www.linkedin.com/in/stadanapisem/\\\">Miljian</a>, </em><a href=\\\"https://www.linkedin.com/in/michal-wegrzyn84/\\\">Micha\u0142</a> and <a href=\\\"https://twitter.com/mmelas74\\\">Marco</a> for their review comments.</p><h4></h4>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"\uD83D\uDC4B Hi, this is Gergely with this month\u2019s free edition of the Pragmatic Engineer Newsletter. In every issue, I cover challenges at big tech and high-growth startups through the lens of engineering managers and senior engineers.\",\"wordcount\":5611,\"post_preview_limit\":null,\"language\":\"en\",\"postTags\":[],\"postCountryBlocks\":[],\"headlineTest\":null,\"coverImagePalette\":{\"Vibrant\":{\"rgb\":[74.57547169811322,180.42452830188677,132.31132075471703],\"population\":0},\"DarkVibrant\":{\"rgb\":[38.77924528301888,93.82075471698113,68.80188679245286],\"population\":0},\"LightVibrant\":{\"rgb\":[180,224,204],\"population\":3077},\"Muted\":{\"rgb\":[116,142,130],\"population\":156},\"DarkMuted\":{\"rgb\":[44.74528301886793,108.25471698113206,79.38679245283022],\"population\":0},\"LightMuted\":{\"rgb\":[172,212,194],\"population\":109}},\"publishedBylines\":[{\"id\":30107029,\"name\":\"Gergely Orosz\",\"handle\":\"pragmaticengineer\",\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/58fed27c-f331-4ff3-ba47-135c5a0be0ba_400x400.png\",\"bio\":\"Big Tech and startups from the inside. Especially relevant for software engineers / AI engineers, useful for anyone working in tech.\",\"profile_set_up_at\":\"2021-09-06T16:08:47.417Z\",\"reader_installed_at\":\"2022-03-04T20:04:29.381Z\",\"publicationUsers\":[{\"id\":385140,\"user_id\":30107029,\"publication_id\":458709,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":458709,\"name\":\"The Pragmatic Engineer\",\"subdomain\":\"pragmaticengineer\",\"custom_domain\":\"newsletter.pragmaticengineer.com\",\"custom_domain_optional\":false,\"hero_text\":\"Big Tech and startups, from the inside. Highly relevant for software engineers, AI engineers and engineering leaders, useful for those working in tech.\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/27076b05-e3f4-4d05-9621-3da4f7f189cd_300x300.png\",\"author_id\":30107029,\"primary_user_id\":30107029,\"theme_var_background_pop\":\"#FF6B00\",\"created_at\":\"2021-08-25T13:08:12.798Z\",\"email_from_name\":\"The Pragmatic Engineer\",\"copyright\":\"Gergely Orosz\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"enabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"newspaper\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"twitter_screen_name\":\"GergelyOrosz\",\"is_guest\":false,\"bestseller_tier\":10000,\"status\":{\"bestsellerTier\":10000,\"subscriberTier\":1,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"bestseller\",\"tier\":10000},\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":101,\"comment_count\":2,\"child_comment_count\":2,\"audio_items\":[{\"post_id\":42747309,\"voice_id\":\"en-US-OnyxTurboMultilingualNeural\",\"audio_url\":\"https://substack-video.s3.amazonaws.com/video_upload/post/42747309/tts/57136797-ffd0-4655-88fa-f4811614b35e/en-US-OnyxTurboMultilingualNeural.mp3\",\"type\":\"tts\",\"status\":\"completed\"}],\"is_geoblocked\":false,\"sponsorshipCampaigns\":{},\"hasCashtag\":false,\"section\":{\"id\":132248,\"created_at\":\"2024-06-18T15:49:33.192Z\",\"updated_at\":\"2024-06-18T16:23:15.198Z\",\"publication_id\":458709,\"name\":\"Deepdives\",\"description\":\"Long-form educational articles every Tuesday. These articles typically go deep into a topic relevant for software engineers or engineering managers, and tend to offer actionable advice.\",\"slug\":\"deepdives\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":2,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"podcastSettings\":null,\"unifiedPodcastSettings\":null,\"showLinks\":[],\"pageTheme\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}},\"comments\":[{\"id\":3366325,\"body\":\"This is a great topic and something in my organisation we have thought about a lot on improving. One progress we've introduced that has helped to onboard people into incident handling is to put together a mock incident. We take a previous incident and run through a war game with a set of engineers to try and discover what the problem is/was. We have a manager playing the role of an incident manager in a mock slack room.\\n\\nThis has been a good tool for onboarding people onto the process, as usually, the first experience is an issue live in production which can be a daunting task.\",\"body_json\":null,\"publication_id\":458709,\"post_id\":42747309,\"user_id\":49909944,\"ancestor_path\":\"\",\"type\":\"comment\",\"deleted\":false,\"date\":\"2021-10-26T13:40:59.826Z\",\"edited_at\":null,\"status\":\"published\",\"pinned_by_user_id\":null,\"restacks\":0,\"name\":\"Christopher Cartlidge Brown\",\"photo_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/b0f0bd21-e89d-4c0f-b544-a0dbe94373c2_144x144.png\",\"handle\":null,\"reactor_names\":[\"Gergely Orosz\"],\"reaction\":null,\"reactions\":{\"\u2764\":2},\"reaction_count\":2,\"children\":[],\"bans\":[],\"suppressed\":false,\"user_banned\":false,\"user_banned_for_comment\":false,\"user_slug\":\"49909944-christopher-cartlidge-brown\",\"metadata\":{\"is_author\":false,\"membership_state\":null,\"eligibleForGift\":true},\"user_bestseller_tier\":null,\"can_dm\":true,\"userStatus\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null},\"score\":7,\"children_count\":0,\"reported_by_user\":false,\"restacked\":false},{\"id\":3292674,\"body\":\"Thank you for compiling all this information. I learned a lot about incident management in my last job and am working to improve incident management in my current job, so this is especially relevant to me in my current role.\\n\\nOne thing I didn't see mentioned is analyzing groups of incidents for commonalities and trends. For instance, one analysis I did showed that a large percentage of our high severity incidents were due to problems with scalability. Because we didn't understand the drivers of scalability, we would run out of a given resource and that would cause an incident. After each incident, we had a post-mortem, but we didn't observe and address the overall trend.\\n\\nAlso, Commonplace (a newsletter run by a guy named Cedric) has several articles on extracting tacit knowledge. Sources of Power by Gary Klein also has insights regarding how people develop and use tacit knowledge.\",\"body_json\":null,\"publication_id\":458709,\"post_id\":42747309,\"user_id\":210211,\"ancestor_path\":\"\",\"type\":\"comment\",\"deleted\":false,\"date\":\"2021-10-19T18:18:50.182Z\",\"edited_at\":null,\"status\":\"published\",\"pinned_by_user_id\":null,\"restacks\":0,\"name\":\"Elaine May\",\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/ccae59ec-99f3-470f-90ff-bb4ba5e564bc_144x144.png\",\"handle\":\"elainemay1\",\"reactor_names\":[\"Gergely Orosz\"],\"reaction\":null,\"reactions\":{\"\u2764\":2},\"reaction_count\":2,\"children\":[],\"bans\":[],\"suppressed\":false,\"user_banned\":false,\"user_banned_for_comment\":false,\"user_slug\":\"elainemay1\",\"metadata\":{\"is_author\":false,\"membership_state\":\"free_signup\",\"eligibleForGift\":true,\"author_on_other_pub\":{\"name\":\"Elaine\u2019s Substack\",\"id\":3962863,\"base_url\":\"https://elainelmay.substack.com\"}},\"user_bestseller_tier\":null,\"can_dm\":true,\"userStatus\":{\"bestsellerTier\":null,\"subscriberTier\":1,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"subscriber\",\"tier\":1,\"accent_colors\":null},\"subscriber\":null},\"score\":7,\"children_count\":0,\"reported_by_user\":false,\"restacked\":false}],\"canonicalUrl\":\"https://newsletter.pragmaticengineer.com/p/incident-review-best-practices\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://newsletter.pragmaticengineer.com/p/incident-review-best-practices\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":\"#FF6B00\",\"background_pop\":\"#FF6B00\",\"color_theme_bg_web\":null,\"cover_bg_color\":\"#FFFFFF\",\"cover_bg_color_secondary\":\"#f0f0f0\",\"background_pop_darken\":\"#e66000\",\"print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_darken\":\"#e66000\",\"color_theme_print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_20\":\"rgba(255, 107, 0, 0.2)\",\"color_theme_bg_pop_30\":\"rgba(255, 107, 0, 0.3)\",\"print_pop\":\"#ff6b00\",\"color_theme_accent\":\"#ff6b00\",\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#ff6b00\",\"font_family_headings_preset\":\"Lora,sans-serif\",\"font_weight_headings_preset\":600,\"font_family_body_preset\":\"'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'\",\"font_weight_body_preset\":400,\"font_preset_heading\":\"fancy_serif\",\"font_preset_body\":\"sans\",\"home_hero\":\"feature-media\",\"home_posts\":\"list\",\"web_bg_color\":\"#ffffff\",\"background_contrast_1\":\"#f0f0f0\",\"background_contrast_2\":\"#dddddd\",\"background_contrast_3\":\"#b7b7b7\",\"background_contrast_4\":\"#929292\",\"background_contrast_5\":\"#515151\",\"color_theme_bg_contrast_1\":\"#f0f0f0\",\"color_theme_bg_contrast_2\":\"#dddddd\",\"color_theme_bg_contrast_3\":\"#b7b7b7\",\"color_theme_bg_contrast_4\":\"#929292\",\"color_theme_bg_contrast_5\":\"#515151\",\"color_theme_bg_elevated\":\"#ffffff\",\"color_theme_bg_elevated_secondary\":\"#f0f0f0\",\"color_theme_bg_elevated_tertiary\":\"#dddddd\",\"color_theme_detail\":\"#e6e6e6\",\"background_contrast_pop\":\"rgba(255, 107, 0, 0.4)\",\"color_theme_bg_contrast_pop\":\"rgba(255, 107, 0, 0.4)\",\"theme_bg_is_dark\":\"0\",\"print_on_web_bg_color\":\"#363737\",\"print_secondary_on_web_bg_color\":\"#868787\",\"background_pop_rgb\":\"255, 107, 0\",\"color_theme_bg_pop_rgb\":\"255, 107, 0\",\"color_theme_accent_rgb\":\"255, 107, 0\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":null,\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
<script nonce="prTDrEptRhTGJ_GBwV-4sg">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"pragmaticengineer\",\"publication_id\":458709,\"has_plans\":true,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"section_name\":\"Deepdives\",\"section_slug\":\"deepdives\",\"section_id\":132248,\"section_is_podcast\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.9b373ce5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.0ebe0daa.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.bc0d221f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.bd0aaae3.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.20d83871.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.e8547d2b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.b215b0b2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.7d252dba.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.0cfb6eb6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.a63c900c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.f1706632.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.0994c2e9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.e7bd7ac9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.c0688e40.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.58c31350.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.ba7bc54b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.ec63b586.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.81261362.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.7ed610b8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.07230a6f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.39093a20.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.7abf0bc6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.42fc8a7f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.97cd0b68.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.d4763852.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.3883839c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89356.5a02543c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.2f9c2d14.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47925.8538b88d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.38bcb610.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71505.5dc348c4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.735979a7.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.56aae81d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.11103ee6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/86118.b1e73de0.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.a6a16056.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.39c246b1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.bae5f6db.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/29231.8e108000.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.90579427.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.81370663.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.9b8aa62d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.71eaa35c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.d6d33385.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.3f14e66b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.510d7dc5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.db11d044.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.0924e824.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.dd9f94bc.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.b05b7730.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.cf9ea6ce.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.a7add5f6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.b6716062.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.a5056440.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.1ae9cf59.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.ccf6a87b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58483.7e92794e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.13a8840c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.d0394e6a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.8b59d68a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/92610.61296cba.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.14a9d722.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11244.f6427483.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99260.94740eff.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.c6c98d54.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/76768.e0df629d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/41932.f923c45b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12114.94e078e5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.01799f30.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.4b96b617.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53871.83f57b67.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.444d3330.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99068.3d46c6e2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.8dd38c51.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.6bd7bf04.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.27c9a7a6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.6debccbf.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.563a346c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.e5bfdc14.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.e47e151d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/32552.50b14e34.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.c1343e93.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.12fa0775.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.f20b0357.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.b928f201.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.ccc81ec4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.1f3fe2f2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.9f376f29.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.0cf66593.js" charset="utf-8"></script>
<script nonce="prTDrEptRhTGJ_GBwV-4sg" nomodule>
(function() {
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
var warningDiv = document.createElement('div');
warningDiv.style.color = 'red';
warningDiv.style.padding = '10px';
warningDiv.style.margin = '10px 0';
warningDiv.style.border = '1px solid red';
warningDiv.style.backgroundColor = 'lightyellow';
warningDiv.innerText = message;
document.body.prepend(warningDiv);
})();
</script>
<!-- Datadog Analytics -->
<script nonce="prTDrEptRhTGJ_GBwV-4sg">
(function(h,o,u,n,d) {
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
d=o.createElement(u);d.async=1;d.src=n
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
window.DD_RUM.onReady(function() {
window.DD_RUM.init({
clientToken: 'puba71073f072643721169b68f352438710',
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
site: 'datadoghq.com',
service: 'web',
env: window._preloads.dd_env,
version: '1ecb4413ae9d7286233b6e6360a9d0a76861b437',
sessionSampleRate: 1,
sessionReplaySampleRate: 10,
trackUserInteractions: window._preloads.dd_ti,
trackResources: true,
trackLongTasks: true,
defaultPrivacyLevel: 'mask-user-input',
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
});
})
</script>
<!-- End Datadog Analytics -->
<!-- Cloudflare Web Analytics -->
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
<!-- End Cloudflare Web Analytics -->
<!-- Fallback tracking pixels -->
<noscript>
<style>
#nojs-banner {
position: fixed;
bottom: 0;
left: 0;
padding: 16px 16px 16px 32px;
width: 100%;
box-sizing: border-box;
background: red;
color: white;
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
font-size: 13px;
line-height: 13px;
}
#nojs-banner a {
color: inherit;
text-decoration: underline;
}
</style>
<div id="nojs-banner">
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
</div>
</noscript>
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="prTDrEptRhTGJ_GBwV-4sg" data-cf-beacon='{"version":"2024.11.0","token":"68cfe66b5c4749e2ba64d4d9640c04c0","spa":2}' crossorigin="anonymous"></script>
</body>
</html>