489 lines
147 KiB
HTML
489 lines
147 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en" dir="ltr">
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
|
||
|
||
|
||
<link rel="preconnect" href="https://substackcdn.com" />
|
||
|
||
|
||
|
||
<title data-rh="true">How GitLab Lost 300GB of Production Data and What We Can Learn</title>
|
||
|
||
<meta data-rh="true" name="theme-color" content="#ffffff"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="How GitLab Lost 300GB of Production Data and What We Can Learn"/><meta data-rh="true" name="twitter:title" content="How GitLab Lost 300GB of Production Data and What We Can Learn"/><meta data-rh="true" name="description" content="The Hard Lesson on Backups, Disaster Recovery, and Human Error"/><meta data-rh="true" property="og:description" content="The Hard Lesson on Backups, Disaster Recovery, and Human Error"/><meta data-rh="true" name="twitter:description" content="The Hard Lesson on Backups, Disaster Recovery, and Human Error"/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!D5yt!,w_1200,h_675,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0d54e327-71cc-4852-8249-5b2e7029fc30_5599x3733.jpeg"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!JdQz!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fbytesizeddesign.substack.com%2Fapi%2Fv1%2Fpost_preview%2F156355259%2Ftwitter.jpg%3Fversion%3D4"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" property="article:modified_time" content="2026-08-30T18:48:22.876Z"/>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@layer legacy, pencraftReset, pencraft;
|
||
</style>
|
||
|
||
|
||
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
|
||
|
||
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/color_links.9856f2f08353fdd05959.css" />
|
||
|
||
|
||
|
||
<link rel="preload" as="font" href="https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwr7I_FMl_E.woff2" crossorigin />
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.cee9fbee.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/86118.c25b9f67.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/76768.f15fc605.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/41932.97d2663c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/32552.313415d5.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
|
||
<meta name="author" content="Byte-Sized Design" />
|
||
<meta property="og:url" content="https://read.bytesizeddesign.com/p/how-gitlab-lost-300gb-of-production" />
|
||
|
||
|
||
<link rel="canonical" href="https://read.bytesizeddesign.com/p/how-gitlab-lost-300gb-of-production" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="shortcut icon" href="https://substackcdn.com/image/fetch/$s_!-fom!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Ffavicon.ico">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="16x16" href="https://substackcdn.com/image/fetch/$s_!T81t!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Ffavicon-16x16.png">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="32x32" href="https://substackcdn.com/image/fetch/$s_!C3AK!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Ffavicon-32x32.png">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="48x48" href="https://substackcdn.com/image/fetch/$s_!RhIY!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Ffavicon-48x48.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="57x57" href="https://substackcdn.com/image/fetch/$s_!W-zx!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-57x57.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="60x60" href="https://substackcdn.com/image/fetch/$s_!w-rM!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-60x60.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="72x72" href="https://substackcdn.com/image/fetch/$s_!7ELy!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-72x72.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="76x76" href="https://substackcdn.com/image/fetch/$s_!d1pX!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-76x76.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="114x114" href="https://substackcdn.com/image/fetch/$s_!Rse_!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-114x114.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="120x120" href="https://substackcdn.com/image/fetch/$s_!dmqP!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-120x120.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="144x144" href="https://substackcdn.com/image/fetch/$s_!gRuL!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-144x144.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="152x152" href="https://substackcdn.com/image/fetch/$s_!7-I-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-152x152.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="167x167" href="https://substackcdn.com/image/fetch/$s_!Nz60!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-167x167.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="180x180" href="https://substackcdn.com/image/fetch/$s_!ZsYi!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-180x180.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="1024x1024" href="https://substackcdn.com/image/fetch/$s_!Ng_l!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F70a352c0-789b-45de-921a-98f6244631f0%2Fapple-touch-icon-1024x1024.png">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="alternate" type="application/rss+xml" href="/feed" title="Byte-Sized Design"/>
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@font-face{font-family:'Lora';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QI8MX1D_JOuMw_hLdO6T2wV9KnW-MoFoqJ2nPWc3ZyhTjcV.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Lora';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QI8MX1D_JOuMw_hLdO6T2wV9KnW-MoFoqt2nPWc3ZyhTjcV.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Lora';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QI8MX1D_JOuMw_hLdO6T2wV9KnW-MoFoqB2nPWc3ZyhTjcV.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Lora';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QI8MX1D_JOuMw_hLdO6T2wV9KnW-MoFoqF2nPWc3ZyhTjcV.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Lora';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QI8MX1D_JOuMw_hLdO6T2wV9KnW-MoFoq92nPWc3ZyhTg.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Lora';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwf7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Lora';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMw77I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Lora';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwX7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Lora';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwT7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Lora';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwr7I_FMl_E.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Lora';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwf7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Lora';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMw77I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Lora';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwX7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Lora';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwT7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Lora';font-style:normal;font-weight:500;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwr7I_FMl_E.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Lora';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwf7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Lora';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMw77I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Lora';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwX7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Lora';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwT7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Lora';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwr7I_FMl_E.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Lora';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwf7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Lora';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMw77I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Lora';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwX7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Lora';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwT7I_FMl_GW8g.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Lora';font-style:normal;font-weight:700;font-display:fallback;src:url(https://fonts.gstatic.com/s/lora/v37/0QIvMX1D_JOuMwr7I_FMl_E.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||
|
||
</style>
|
||
|
||
|
||
|
||
|
||
|
||
<style data-pub-theme-prefix>:root{--color_theme_bg_pop:#7671ec;--background_pop:#7671ec;--color_theme_bg_web:#ffffff;--cover_bg_color:#ffffff;--cover_bg_color_secondary:#f0f0f0;--background_pop_darken:#605be9;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#605be9;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(118, 113, 236, 0.2);--color_theme_bg_pop_30:rgba(118, 113, 236, 0.3);--print_pop:#7671ec;--color_theme_accent:#7671ec;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#7671ec;--font_family_headings_preset:'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';--font_weight_headings_preset:700;--font_family_body_preset:Lora,sans-serif;--font_weight_body_preset:400;--font_size_body_offset:1px;--font_preset_heading:sans;--font_preset_body:fancy_serif;--home_hero:magazine-5;--home_posts:list;--home_show_top_posts:true;--web_bg_color:#ffffff;--background_contrast_1:#f0f0f0;--background_contrast_2:#dddddd;--background_contrast_3:#b7b7b7;--background_contrast_4:#929292;--background_contrast_5:#515151;--color_theme_bg_contrast_1:#f0f0f0;--color_theme_bg_contrast_2:#dddddd;--color_theme_bg_contrast_3:#b7b7b7;--color_theme_bg_contrast_4:#929292;--color_theme_bg_contrast_5:#515151;--color_theme_bg_elevated:#ffffff;--color_theme_bg_elevated_secondary:#f0f0f0;--color_theme_bg_elevated_tertiary:#dddddd;--color_theme_detail:#e6e6e6;--background_contrast_pop:rgba(118, 113, 236, 0.4);--color_theme_bg_contrast_pop:rgba(118, 113, 236, 0.4);--theme_bg_is_dark:0;--print_on_web_bg_color:#363737;--print_secondary_on_web_bg_color:#868787;--background_pop_rgb:118, 113, 236;--color_theme_bg_pop_rgb:118, 113, 236;--color_theme_accent_rgb:118, 113, 236;}</style>
|
||
|
||
|
||
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
|
||
|
||
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/color_links.9856f2f08353fdd05959.css" />
|
||
|
||
|
||
<style data-pub-theme-postfix></style>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</head>
|
||
|
||
<body class="">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div id="entry">
|
||
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:71px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="flex-basis:0px;flex-grow:1;" class="logoContainer-p12gJb"><a href="/" native class="pencraft pc-display-contents pc-reset"><div draggable="false" class="pencraft pc-display-flex pc-position-relative pc-reset"><div style="width:40px;height:40px;" class="pencraft pc-display-flex pc-reset bg-white-ZBV5av pc-borderRadius-sm overflow-hidden-WdpwT6 sizing-border-box-DggLA4"><picture class="pencraft pc-display-contents pc-reset"><source type="image/webp" sizes="40px" srcset="https://substackcdn.com/image/fetch/$s_!UMZA!,w_40,h_40,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png 40w, https://substackcdn.com/image/fetch/$s_!UMZA!,w_80,h_80,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png 80w, https://substackcdn.com/image/fetch/$s_!UMZA!,w_120,h_120,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png 120w" style="display:none;"/><img style="object-fit:cover;width:40px;height:40px;" src="https://substackcdn.com/image/fetch/$s_!UMZA!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png" srcset="https://substackcdn.com/image/fetch/$s_!UMZA!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png 40w, https://substackcdn.com/image/fetch/$s_!UMZA!,w_80,h_80,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png 80w, https://substackcdn.com/image/fetch/$s_!UMZA!,w_120,h_120,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png 120w" sizes="40px" alt="Byte-Sized Design" draggable="false" class="pencraft pc-reset"/></picture></div></div></a></div><div style="flex-grow:0;" class="titleContainer-DJYq5v"><h1 class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd"><a href="/" class="pencraft pc-display-contents pc-reset">Byte-Sized Design</a></h1></div><div style="flex-basis:0px;flex-grow:1;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Fhow-gitlab-lost-300gb-of-production%3Futm_campaign%3Dpost%26utm_medium%3Dweb&for_pub=bytesizeddesign" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="height:72px;"></div></div></div><div><script nonce="xBzf9d9OeqN6zCc0ysemKQ" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://read.bytesizeddesign.com/p/how-gitlab-lost-300gb-of-production","mainEntityOfPage":"https://read.bytesizeddesign.com/p/how-gitlab-lost-300gb-of-production","headline":"How GitLab Lost 300GB of Production Data and What We Can Learn","description":"The Hard Lesson on Backups, Disaster Recovery, and Human Error","image":[{"@type":"ImageObject","url":"https://substack-post-media.s3.amazonaws.com/public/images/0d54e327-71cc-4852-8249-5b2e7029fc30_5599x3733.jpeg"}],"datePublished":"2025-02-03T06:36:13+00:00","dateModified":"2025-02-03T06:36:13+00:00","isAccessibleForFree":false,"author":[{"@type":"Person","name":"Byte-Sized Design","url":"https://substack.com/@bytesizeddesign","description":"Real talk about building systems that work. Career advice for growing engineers, how actual companies solve large-scale problems, post-mortems from things that broke, and the system design fundamentals that matter. No fluff, no buzzwords.","identifier":"user:104689928","image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!XD3N!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!XD3N!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png"}}],"publisher":{"@type":"Organization","name":"Byte-Sized Design","url":"https://read.bytesizeddesign.com","description":"Master system design concepts, engineering fundamentals, and interview basics. Weekly summaries, post-mortems, and advice for 43,000+ engineers.","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":10000},"identifier":"pub:1499683","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!UMZA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!UMZA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!UMZA!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!UMZA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!UMZA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!UMZA!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png"},"sameAs":["https://twitter.com/alexcancode"]},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":43},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":6},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":0}]}</script><script nonce="xBzf9d9OeqN6zCc0ysemKQ" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Byte-Sized Design","item":"https://read.bytesizeddesign.com"},{"@type":"ListItem","position":2,"name":"How GitLab Lost 300GB of Production Data and What We Can Learn","item":"https://read.bytesizeddesign.com/p/how-gitlab-lost-300gb-of-production"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><h1 dir="auto" class="post-title published title-X77sOw">How GitLab Lost 300GB of Production Data and What We Can Learn</h1><h3 dir="auto" class="subtitle subtitle-HEEcLo">The Hard Lesson on Backups, Disaster Recovery, and Human Error</h3><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-row pc-gap-8 pc-alignItems-center pc-justifyContent-flex-start pc-reset"><div style="--scale:36px;--offset:9px;--border-width:4.5px;" class="pencraft pc-display-flex pc-flexDirection-row pc-alignItems-center pc-justifyContent-flex-start pc-reset ltr-qDBmby"><a href="https://substack.com/@bytesizeddesign" aria-label="View Byte-Sized Design's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:36px;" tabindex="0" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6 avatar-u8q6xB last-JfNEJ_"><div style="--scale:36px;" title="Byte-Sized Design" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!XD3N!,w_36,h_36,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 1x, https://substackcdn.com/image/fetch/$s_!XD3N!,w_72,h_72,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 2x, https://substackcdn.com/image/fetch/$s_!XD3N!,w_108,h_108,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!XD3N!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png" alt="Byte-Sized Design's avatar" srcset="https://substackcdn.com/image/fetch/$s_!XD3N!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 1x, https://substackcdn.com/image/fetch/$s_!XD3N!,w_72,h_72,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 2x, https://substackcdn.com/image/fetch/$s_!XD3N!,w_108,h_108,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 3x" width="36" height="36" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-reset color-primary-zABazT line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><span style="min-width:0;" data-state="closed"><a href="https://substack.com/@bytesizeddesign" class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ">Byte-Sized Design</a></span></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2025-02-03T06:36:13.709Z" title="2025-02-03T06:36:13.709Z">Feb 03, 2025</time></div><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA">∙ Paid</div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (43)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">43</div></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://read.bytesizeddesign.com/p/how-gitlab-lost-300gb-of-production/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-27" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">6</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p>On January 31, 2017, GitLab.com went down hard. For hours, engineers scrambled to recover, only to realize their backup system had been broken for weeks. By the time the dust settled, they had lost 5,000 projects, 700 user accounts, and countless issues and comments.</p><p>The root cause? A single mistaken command that wiped their primary database.</p><p>In a post-mortem, GitLab detailed how they lost critical production data, the mistakes that led to the disaster, and the systemic failures that made recovery nearly impossible.</p><p>Let’s break it down:</p><h3 class="header-anchor-post"><span>🚀 </span><strong>TL;DR</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§tldr" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/tldr" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>GitLab.com suffered a major outage after an engineer accidentally </span><strong>deleted the primary database instead of a secondary replica.</strong><span> The disaster was compounded by broken backups, slow recovery tools, and a failure to catch issues before they escalated.</span></p><p>Here’s what we’ll cover:</p><p><span>1️⃣ </span><strong>🛠️ What Happened?</strong><span> (The moment that wiped GitLab’s data)</span><br/><span>2️⃣ </span><strong>🛑 Root Causes</strong><span> (The silent failures that made recovery impossible)</span><br/><span>3️⃣ </span><strong>🤔 Lessons Learned</strong><span> (How GitLab changed its approach)</span><br/><span>4️⃣ </span><strong>🏗️ How to Protect Your Own Systems</strong><span> (Paid)</span></p><p><span>💡 </span><strong>More Insights & Recovery Strategies </strong><span>(Paid)</span></p><div><hr/></div><h2 class="header-anchor-post"><strong>🛠️ What Happened?</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§what-happened" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/what-happened" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><span>It started as a </span><strong>routine maintenance task.</strong></p><p>To test new database scaling techniques, a GitLab engineer needed to resynchronize a secondary PostgreSQL database. The process required wiping its data directory before restoring a fresh copy from the primary database.</p><p><span>🚨 </span><strong>The mistake:</strong><span> Instead of running the command on the secondary server, the engineer executed it on the </span><strong>primary database.</strong></p><p><span>💾 </span><strong>300GB of production data vanished in seconds.</strong></p><p><span>Panic set in. The team turned to their backups—only to find that their </span><strong>backup system had been failing silently for weeks.</strong></p><p><span>With no working backups and a painfully slow recovery process, GitLab had no choice but to </span><strong>rebuild from a 6-hour-old snapshot, permanently losing all data created after that point.</strong></p><div><hr/></div><h2 class="header-anchor-post"><strong>🛑 Root Causes</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§root-causes" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/root-causes" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><h3 class="header-anchor-post"><strong>1️⃣ No Protection Against Accidental Deletion</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§1-no-protection-against-accidental-deletion" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/1-no-protection-against-accidental-deletion" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p>There was no safeguard to prevent engineers from running destructive commands on production systems.</p></li><li><p><span>The primary and secondary databases </span><strong>looked identical</strong><span> in the terminal making it easy to confuse them.</span></p></li></ul><h3 class="header-anchor-post"><strong>2️⃣ Backups Had Been Failing Silently for Weeks</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§2-backups-had-been-failing-silently-for-weeks" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/2-backups-had-been-failing-silently-for-weeks" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p><span>GitLab’s automated </span><code>pg_dump</code><span> backups </span><strong>never actually ran</strong><span> due to a misconfiguration.</span></p></li><li><p><span>Backup failure alerts </span><strong>were sent via email</strong><span>—but those emails were silently rejected due to DMARC settings.</span></p></li></ul><h3 class="header-anchor-post"><strong>3️⃣ The Recovery Process Was Too Slow</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§3-the-recovery-process-was-too-slow" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/3-the-recovery-process-was-too-slow" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p><span>Azure disk snapshots were available but </span><strong>took over 18 hours to restore</strong><span> due to slow disk speeds.</span></p></li><li><p><span>There was no clear, </span><strong>battle-tested disaster recovery</strong><span> process to follow.</span></p></li></ul><h3 class="header-anchor-post"><strong>4️⃣ Replication Lag Left No Secondary Backup</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§4-replication-lag-left-no-secondary-backup" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/4-replication-lag-left-no-secondary-backup" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p><span>The secondary database was out of sync due to a </span><strong>spike in database load</strong><span> earlier that day.</span></p></li><li><p><span>When replication stopped, </span><strong>old transaction logs were purged</strong><span>, meaning the secondary couldn't be used as a failover.</span></p></li></ul><p><strong>In short: GitLab lost production data because of a mix of human error, broken backups, and an untested recovery process.</strong></p><div><hr/></div><h2 class="header-anchor-post"><strong>🤔 Lessons Learned</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§lessons-learned" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/lessons-learned" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><span>GitLab’s failure highlights </span><strong>five critical lessons</strong><span> for any engineering team:</span></p><h3 class="header-anchor-post"><strong>1️⃣ Your Backups Are Useless Until You Test Them</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§1-your-backups-are-useless-until-you-test-them" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/1-your-backups-are-useless-until-you-test-them" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p><span>If you don’t regularly </span><strong>restore from backups</strong><span>, they might not work when you need them.</span></p></li><li><p><strong>GitLab assumed their backups worked.</strong><span> They didn't.</span></p></li></ul><h3 class="header-anchor-post"><strong>2️⃣ Critical Systems Need Guardrails</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§2-critical-systems-need-guardrails" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/2-critical-systems-need-guardrails" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p><span>Running destructive commands on production databases should require </span><strong>extra confirmation steps.</strong></p></li><li><p><span>Engineers should see </span><strong>clear visual indicators</strong><span> showing whether they’re working on production or staging.</span></p></li></ul><h3 class="header-anchor-post"><strong>3️⃣ Recovery Speed Matters More Than Backup Frequency</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§3-recovery-speed-matters-more-than-backup-frequency" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/3-recovery-speed-matters-more-than-backup-frequency" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p><span>Even if backups exist, if it takes </span><strong>18+ hours to restore</strong><span>, your business will suffer.</span></p></li><li><p><strong>Solution:</strong><span> Invest in </span><strong>point-in-time recovery (PITR)</strong><span> instead of relying solely on daily snapshots.</span></p></li></ul><h3 class="header-anchor-post"><strong>4️⃣ Test Disaster Recovery Like You Test Your Code</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§4-test-disaster-recovery-like-you-test-your-code" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/4-test-disaster-recovery-like-you-test-your-code" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p><span>Have a </span><strong>documented, rehearsed plan</strong><span> for recovering from catastrophic failure.</span></p></li><li><p><span>Run </span><strong>fire drills</strong><span> where engineers simulate total database loss and restore from backups.</span></p></li><li><p><span>Build automated chaos tests when and where possible throughout your pipelines. </span><a href="https://bytesizeddesign.substack.com/p/the-engineers-guide-to-observability">Read more here</a><span>.</span></p><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div style="margin:20px 0;" data-component-name="CommentViewPlaceholder" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-center pc-alignItems-center pc-reset"><div style="text-decoration:none;width:560px;max-width:100%;" class="pencraft pc-display-flex pc-gap-12 pc-padding-20 pc-mobile-padding-16 pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-md note-Biuyq7"><div style="width:48px;height:48px;" tabindex="-1" aria-hidden="true" class="pencraft pc-reset pc-borderRadius-full placeholder-ICMYsF"></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-12 pc-reset flex-grow-rzmknG"><div style="width:59.29456345651511%;height:16px;" tabindex="-1" aria-hidden="true" class="pencraft pc-reset pc-borderRadius-sm placeholder-ICMYsF"></div><div style="width:91.59916731570996%;height:16px;" tabindex="-1" aria-hidden="true" class="pencraft pc-reset pc-borderRadius-sm placeholder-ICMYsF"></div><div style="width:85.59892927456215%;height:16px;" tabindex="-1" aria-hidden="true" class="pencraft pc-reset pc-borderRadius-sm placeholder-ICMYsF"></div><div style="width:71.10581008678665%;height:16px;" tabindex="-1" aria-hidden="true" class="pencraft pc-reset pc-borderRadius-sm placeholder-ICMYsF"></div></div></div></div></div></li></ul><h3 class="header-anchor-post"><strong>5️⃣ Redundancy Must Be Truly Redundant</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§5-redundancy-must-be-truly-redundant" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/5-redundancy-must-be-truly-redundant" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><ul><li><p><span>A secondary database that </span><strong>isn’t truly in sync</strong><span> won’t save you in a crisis.</span></p></li><li><p><span>GitLab fixed this by adding </span><strong>multiple secondaries across different regions.</strong></p></li></ul><div><hr/></div><h2 class="header-anchor-post"><strong>🏗️ How to Protect Your Own Systems</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§how-to-protect-your-own-systems" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://read.bytesizeddesign.com/i/156355259/how-to-protect-your-own-systems" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>Many companies have similar risks lurking in their infrastructure. Here’s what you can do to avoid disaster:</p></div></div></div><div data-testid="paywall" data-component-name="Paywall" role="region" aria-label="Paywall" class="paywall"><div style="--scale:64px;" class="pencraft pc-display-flex pc-width-64 pc-height-64 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj paywall-unlock-avatar"><div style="--scale:64px;" title="User" class="pencraft pc-display-flex pc-width-64 pc-height-64 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!XD3N!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 1x, https://substackcdn.com/image/fetch/$s_!XD3N!,w_128,h_128,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 2x, https://substackcdn.com/image/fetch/$s_!XD3N!,w_192,h_192,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!XD3N!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!XD3N!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 1x, https://substackcdn.com/image/fetch/$s_!XD3N!,w_128,h_128,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 2x, https://substackcdn.com/image/fetch/$s_!XD3N!,w_192,h_192,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png 3x" width="64" height="64" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><h2 class="paywall-title unlock-treatment-new">Continue reading this post for free, courtesy of Byte-Sized Design.</h2><div class="paywall-cta"><a data-native="true"><button tabindex="0" type="button" class="pencraft pc-reset pencraft subscribe-btn subscribeButton-LcKYi7 paywall-cta-icon buttonBase-GK1x3M">Claim my free post</button></a></div><div class="paywall-secondary-cta"><a href="https://read.bytesizeddesign.com/subscribe?simple=true&next=https%3A%2F%2Fread.bytesizeddesign.com%2Fp%2Fhow-gitlab-lost-300gb-of-production%3Futm_campaign%3Dpost%26utm_medium%3Dweb&utm_source=paywall&utm_medium=web&utm_content=156355259&just_signed_up=falsesimple=true&utm_source=paywall&utm_medium=email&utm_content=156355259&next=https://read.bytesizeddesign.com/p/how-gitlab-lost-300gb-of-production" data-native="true">Or purchase a paid subscription.</a></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-left"><path d="m12 19-7-7 7-7"></path><path d="M19 12H5"></path></svg>Previous</button><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o">Next<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></button></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer themed-background"><div class="container"><div class="footer-terms"><span>© 2026 Byte-Sized Design</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&utm_medium=web&utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="xBzf9d9OeqN6zCc0ysemKQ">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
|
||
</div>
|
||
|
||
|
||
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
|
||
<script nonce="xBzf9d9OeqN6zCc0ysemKQ">
|
||
window.Sentry && window.Sentry.onLoad(function() {
|
||
window.Sentry.init({
|
||
environment: window._preloads.sentry_environment,
|
||
dsn: window._preloads.sentry_dsn,
|
||
})
|
||
})
|
||
</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script nonce="xBzf9d9OeqN6zCc0ysemKQ">window._preloads = JSON.parse("{\"cspNonce\":\"xBzf9d9OeqN6zCc0ysemKQ\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://read.bytesizeddesign.com\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":104689928,\"byline_images_enabled\":true,\"bylines_enabled\":true,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Byte-Sized Design\",\"cover_photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/65ac4fc6-e2d8-4581-9bfb-86082179418f_1280x320.png\",\"created_at\":\"2023-03-16T21:02:31.841Z\",\"custom_domain_optional\":false,\"custom_domain\":\"read.bytesizeddesign.com\",\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":\"fca7ca70\",\"default_show_guest_bios\":true,\"email_banner_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/d571a9c4-5f7f-4ca2-9884-8a3d0d4f81d4_1584x396.png\",\"email_from_name\":\"Byte-Sized Design\",\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":[],\"free_subscription_benefits\":[\"1 Byte Sized Design post each week, summarizing impactful technical designs to boost your knowledge!\"],\"ga_pixel_id\":null,\"google_site_verification_token\":null,\"google_tag_manager_token\":null,\"hero_image\":null,\"hero_text\":\"Master system design concepts, engineering fundamentals, and interview basics. Weekly summaries, post-mortems, and advice for 43,000+ engineers.\",\"hide_intro_subtitle\":null,\"hide_intro_title\":false,\"hide_podcast_feed_link\":false,\"homepage_type\":\"magaziney\",\"id\":1499683,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":null,\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!UMZA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png\",\"minimum_group_size\":2,\"moderation_enabled\":false,\"name\":\"Byte-Sized Design\",\"paid_subscription_benefits\":[\"Access to the full list of 73 of the most influential engineering blogs!\",\"Exclusive access to our full archive of posts and official article resources!\",\"5 System Design Templates for clear, comprehensive and impactful technical writing!\"],\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"enabled\",\"paywall_free_trial_enabled\":true,\"podcast_art_url\":null,\"paid_podcast_episode_art_url\":null,\"podcast_byline\":null,\"podcast_description\":null,\"podcast_enabled\":false,\"podcast_feed_url\":null,\"podcast_title\":null,\"post_preview_limit\":200,\"primary_user_id\":104689928,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"bytesizeddesign\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#BAA049\",\"theme_var_color_links\":true,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":null,\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":[{\"id\":\"monthly8usd\",\"object\":\"plan\",\"active\":true,\"aggregate_usage\":null,\"amount\":800,\"amount_decimal\":\"800\",\"billing_scheme\":\"per_unit\",\"created\":1773961395,\"currency\":\"usd\",\"interval\":\"month\",\"interval_count\":1,\"livemode\":true,\"metadata\":{\"substack\":\"yes\"},\"meter\":null,\"nickname\":\"$8 a month\",\"product\":\"prod_UBByWx1TYuvrpE\",\"tiers_mode\":null,\"transform_usage\":null,\"trial_period_days\":null,\"usage_type\":\"licensed\",\"currency_options\":{\"aud\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":1200,\"unit_amount_decimal\":\"1200\"},\"brl\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":4100,\"unit_amount_decimal\":\"4100\"},\"cad\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":1200,\"unit_amount_decimal\":\"1200\"},\"chf\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":700,\"unit_amount_decimal\":\"700\"},\"dkk\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":5500,\"unit_amount_decimal\":\"5500\"},\"eur\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":700,\"unit_amount_decimal\":\"700\"},\"gbp\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":600,\"unit_amount_decimal\":\"600\"},\"inr\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":77000,\"unit_amount_decimal\":\"77000\"},\"jpy\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":1400,\"unit_amount_decimal\":\"1400\"},\"mxn\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":14000,\"unit_amount_decimal\":\"14000\"},\"nok\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":8000,\"unit_amount_decimal\":\"8000\"},\"nzd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":1400,\"unit_amount_decimal\":\"1400\"},\"pln\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":3100,\"unit_amount_decimal\":\"3100\"},\"sek\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":8000,\"unit_amount_decimal\":\"8000\"},\"usd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":800,\"unit_amount_decimal\":\"800\"}}},{\"id\":\"yearly80usd\",\"object\":\"plan\",\"active\":true,\"aggregate_usage\":null,\"amount\":8000,\"amount_decimal\":\"8000\",\"billing_scheme\":\"per_unit\",\"created\":1773961396,\"currency\":\"usd\",\"interval\":\"year\",\"interval_count\":1,\"livemode\":true,\"metadata\":{\"substack\":\"yes\"},\"meter\":null,\"nickname\":\"$80 a year\",\"product\":\"prod_UBBy4D4tZ91SDp\",\"tiers_mode\":null,\"transform_usage\":null,\"trial_period_days\":null,\"usage_type\":\"licensed\",\"currency_options\":{\"aud\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":12000,\"unit_amount_decimal\":\"12000\"},\"brl\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":41000,\"unit_amount_decimal\":\"41000\"},\"cad\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":11500,\"unit_amount_decimal\":\"11500\"},\"chf\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":7000,\"unit_amount_decimal\":\"7000\"},\"dkk\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":52500,\"unit_amount_decimal\":\"52500\"},\"eur\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":7000,\"unit_amount_decimal\":\"7000\"},\"gbp\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":6000,\"unit_amount_decimal\":\"6000\"},\"inr\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":766000,\"unit_amount_decimal\":\"766000\"},\"jpy\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":13500,\"unit_amount_decimal\":\"13500\"},\"mxn\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":140000,\"unit_amount_decimal\":\"140000\"},\"nok\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":77000,\"unit_amount_decimal\":\"77000\"},\"nzd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":14000,\"unit_amount_decimal\":\"14000\"},\"pln\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":30500,\"unit_amount_decimal\":\"30500\"},\"sek\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":77500,\"unit_amount_decimal\":\"77500\"},\"usd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":8000,\"unit_amount_decimal\":\"8000\"}}},{\"id\":\"founding15000usd\",\"name\":\"founding15000usd\",\"nickname\":\"founding15000usd\",\"active\":true,\"amount\":15000,\"currency\":\"usd\",\"interval\":\"year\",\"interval_count\":1,\"metadata\":{\"substack\":\"yes\",\"founding\":\"yes\",\"no_coupons\":\"yes\",\"short_description\":\"Founding Member\",\"short_description_english\":\"Founding Member\",\"minimum\":\"8000\",\"minimum_local\":{\"usd\":8000,\"gbp\":6000,\"eur\":7000,\"cad\":11500,\"mxn\":136000,\"aud\":11500,\"brl\":41000,\"sek\":78000,\"chf\":7000,\"dkk\":52000,\"nok\":74500,\"inr\":765000,\"jpy\":12500,\"nzd\":14000,\"pln\":30000}},\"currency_options\":{\"usd\":{\"unit_amount\":15000,\"tax_behavior\":\"unspecified\"},\"gbp\":{\"unit_amount\":11500,\"tax_behavior\":\"unspecified\"},\"eur\":{\"unit_amount\":13000,\"tax_behavior\":\"unspecified\"},\"cad\":{\"unit_amount\":21000,\"tax_behavior\":\"unspecified\"},\"mxn\":{\"unit_amount\":255000,\"tax_behavior\":\"unspecified\"},\"aud\":{\"unit_amount\":21000,\"tax_behavior\":\"unspecified\"},\"brl\":{\"unit_amount\":77000,\"tax_behavior\":\"unspecified\"},\"sek\":{\"unit_amount\":146000,\"tax_behavior\":\"unspecified\"},\"chf\":{\"unit_amount\":12500,\"tax_behavior\":\"unspecified\"},\"dkk\":{\"unit_amount\":97000,\"tax_behavior\":\"unspecified\"},\"nok\":{\"unit_amount\":139500,\"tax_behavior\":\"unspecified\"},\"inr\":{\"unit_amount\":1434000,\"tax_behavior\":\"unspecified\"},\"jpy\":{\"unit_amount\":23500,\"tax_behavior\":\"unspecified\"},\"nzd\":{\"unit_amount\":26000,\"tax_behavior\":\"unspecified\"},\"pln\":{\"unit_amount\":56000,\"tax_behavior\":\"unspecified\"}}}],\"stripe_user_id\":\"acct_1MnT6HEC2sPvm4nA\",\"stripe_country\":\"US\",\"stripe_publishable_key\":\"pk_live_51MnT6HEC2sPvm4nAh3sFiolMwQytPWddOPdAWOXzBtu89DjpdpL127ao6iW8YRHOLS1uBMo78voPqrhYB49o3FdJ00v0B7vTjJ\",\"stripe_platform_account\":\"US\",\"automatic_tax_enabled\":false,\"author_name\":\"Byte-Sized Design\",\"author_handle\":\"bytesizeddesign\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!XD3N!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png\",\"author_bio\":\"Real talk about building systems that work. Career advice for growing engineers, how actual companies solve large-scale problems, post-mortems from things that broke, and the system design fundamentals that matter. No fluff, no buzzwords.\",\"twitter_screen_name\":\"alexcancode\",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":\"#7671ec\",\"web_bg_color\":\"#ffffff\",\"cover_bg_color\":\"#ffffff\",\"publication_id\":1499683,\"color_links\":null,\"font_preset_heading\":\"sans\",\"font_preset_body\":\"fancy_serif\",\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"magazine-5\",\"home_posts\":\"list\",\"home_show_top_posts\":true,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":null,\"custom_footer\":null,\"social_media_links\":null,\"font_options\":null,\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":{\"first_thread_email_sent_at\":null,\"activated_at\":null,\"reader_thread_notifications_enabled\":true,\"create_thread_minimum_role\":\"contributor\",\"photo_replies_enabled\":true,\"boost_free_subscriber_chat_preview_enabled\":false,\"push_suppression_enabled\":false},\"default_group_coupon_percent_off\":\"20.00\",\"default_group_coupon_include_founding\":false,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2023-03-19T22:53:43.956Z\",\"has_podcast\":false,\"has_free_podcast\":false,\"has_subscriber_only_podcast\":false,\"has_community_content\":true,\"rankingDetail\":\"Hundreds of paid subscribers\",\"rankingDetailFreeIncluded\":\"Tens of thousands of subscribers\",\"rankingDetailOrderOfMagnitude\":100,\"rankingDetailFreeIncludedOrderOfMagnitude\":10000,\"rankingDetailFreeSubscriberCount\":\"Over 43,000 subscribers\",\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u0645\u0626\u0627\u062A \u0627\u0644\u0645\u0634\u062A\u0631\u0643\u064A\u0646 \u0627\u0644\u0645\u062F\u0641\u0648\u0639\u064A\u0646\"},\"ca\":{\"rankingDetail\":\"Centenars de subscriptors de pagament\"},\"da\":{\"rankingDetail\":\"Hundredvis af betalte abonnenter\"},\"de\":{\"rankingDetail\":\"Hunderte von Paid-Abonnenten\"},\"es\":{\"rankingDetail\":\"Cientos de suscriptores de pago\"},\"fr\":{\"rankingDetail\":\"Des centaines d'abonn\u00E9s payants\"},\"id\":{\"rankingDetail\":\"Ratusan pelanggan berbayar\"},\"ja\":{\"rankingDetail\":\"\u6570\u767E\u4EBA\u306E\u6709\u6599\u767B\u9332\u8005\"},\"ko\":{\"rankingDetail\":\"\uC218\uBC31 \uBA85\uC758 \uC720\uB8CC \uAD6C\uB3C5\uC790\"},\"nb\":{\"rankingDetail\":\"Hundrevis av betalende abonnenter\"},\"nl\":{\"rankingDetail\":\"Honderden betalende abonnees\"},\"pl\":{\"rankingDetail\":\"Setki p\u0142ac\u0105cych subskrybent\u00F3w\"},\"pt\":{\"rankingDetail\":\"Centenas de subscritores pagos\"},\"pt-br\":{\"rankingDetail\":\"Centenas de assinantes pagantes\"},\"en-gb\":{\"rankingDetail\":\"Hundreds of paid subscribers\"},\"it\":{\"rankingDetail\":\"Centinaia di abbonati a pagamento\"},\"tr\":{\"rankingDetail\":\"Y\u00FCzlerce \u00FCcretli abone\"},\"sv\":{\"rankingDetail\":\"Hundratals betalande prenumeranter\"},\"fi\":{\"rankingDetail\":\"Hundreds of paid subscribers\"},\"is\":{\"rankingDetail\":\"Hundreds of paid subscribers\"},\"zh-hans\":{\"rankingDetail\":\"\u6570\u767E\u540D\u4ED8\u8D39\u8BA2\u9605\u8005\"},\"zh-hant\":{\"rankingDetail\":\"\u6578\u767E\u4F4D\u4ED8\u8CBB\u8A02\u95B1\u8005\"},\"en\":{\"rankingDetail\":\"Hundreds of paid subscribers\"}},\"freeSubscriberCount\":\"43,000\",\"freeSubscriberCountOrderOfMagnitude\":\"43K+\",\"author_bestseller_tier\":100,\"author_badge\":{\"type\":\"bestseller\",\"tier\":100},\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":true,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":null,\"primary_profile_name\":\"Byte-Sized Design\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!XD3N!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png\",\"no_follow\":false,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"paid\",\"sections\":[],\"podcastTabInfo\":null,\"multipub_migration\":null,\"navigationBarItems\":[{\"id\":\"83256cbb-f820-4c94-a126-6cbaa5eb50bb\",\"publication_id\":1499683,\"sibling_rank\":0,\"link_title\":\"Sponsorships\",\"link_url\":\"https://www.passionfroot.me/byte-sized-design\",\"section_id\":null,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"section\":null,\"postTag\":null,\"children\":[],\"post\":null},{\"id\":\"a6130bf5-2ba6-4b52-8876-34872dabfde7\",\"publication_id\":1499683,\"sibling_rank\":1,\"link_title\":\"Paid Resources \",\"link_url\":\"\",\"section_id\":null,\"post_id\":151597571,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"section\":null,\"postTag\":null,\"children\":[],\"post\":{\"id\":151597571,\"slug\":\"paid-resources\",\"title\":\"Paid Resources \",\"type\":\"page\"}}],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Byte-Sized Design\",\"handle\":\"bytesizeddesign\",\"role\":\"admin\",\"owner\":true,\"user_id\":104689928,\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png\",\"bio\":\"Real talk about building systems that work. Career advice for growing engineers, how actual companies solve large-scale problems, post-mortems from things that broke, and the system design fundamentals that matter. No fluff, no buzzwords.\",\"status\":{\"bestsellerTier\":100,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"bestseller\",\"tier\":100},\"subscriber\":null}}],\"threads_v2_enabled\":true,\"viralGiftsConfig\":{\"id\":\"959ff0a7-8b3a-42ab-a3b7-c689eab5e738\",\"publication_id\":1499683,\"enabled\":true,\"gifts_per_user\":5,\"gift_length_months\":1,\"send_extra_gifts\":true,\"message\":\"My personal Substack\",\"created_at\":\"2023-03-19T20:50:09.58833+00:00\",\"updated_at\":\"2023-03-19T20:50:09.58833+00:00\",\"days_til_invite\":14,\"send_emails\":true,\"show_link\":null},\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"iap_advanced_plans\":[{\"sku\":\"efFyTkosIej3GhkPVO\",\"publication_id\":\"1499683\",\"is_active\":true,\"price_base_units\":1100,\"currency_alpha3\":\"usd\",\"period\":\"month\",\"created_at\":\"2026-03-19T23:03:18.177Z\",\"updated_at\":\"2026-03-19T23:03:18.177Z\",\"id\":\"1011617\",\"payout_amount_base_units\":80,\"alternate_currencies\":{\"aud\":1600,\"brl\":6000,\"cad\":1600,\"chf\":900,\"dkk\":7500,\"eur\":1000,\"gbp\":900,\"mxn\":20000,\"nok\":10500,\"nzd\":1900,\"pln\":4100,\"sek\":10500},\"is_founding\":false,\"display_name\":\"Byte-Sized Design (Monthly)\",\"display_price\":\"$11\"},{\"sku\":\"RPJGMy7hmFpkSvHvfB\",\"publication_id\":\"1499683\",\"is_active\":true,\"price_base_units\":11000,\"currency_alpha3\":\"usd\",\"period\":\"year\",\"created_at\":\"2026-03-19T23:03:18.192Z\",\"updated_at\":\"2026-03-19T23:03:18.192Z\",\"id\":\"1011618\",\"payout_amount_base_units\":800,\"alternate_currencies\":{\"aud\":16000,\"brl\":57500,\"cad\":15500,\"chf\":9000,\"dkk\":71000,\"eur\":9500,\"gbp\":8500,\"mxn\":195500,\"nok\":104500,\"nzd\":19000,\"pln\":41000,\"sek\":102500},\"is_founding\":false,\"display_name\":\"Byte-Sized Design (Yearly)\",\"display_price\":\"$110\"}],\"iap_founding_plan\":{\"base_plan_id\":\"founding15000usd\",\"name\":\"Founding Member\",\"minimum_amount\":11000,\"suggested_amount\":20500,\"currency_alpha3\":\"usd\",\"alternate_currencies\":{\"gbp\":{\"minimum_amount\":8500,\"suggested_amount\":15500},\"eur\":{\"minimum_amount\":9500,\"suggested_amount\":18000},\"cad\":{\"minimum_amount\":15500,\"suggested_amount\":28500},\"mxn\":{\"minimum_amount\":187000,\"suggested_amount\":348000},\"aud\":{\"minimum_amount\":15500,\"suggested_amount\":29000},\"brl\":{\"minimum_amount\":56500,\"suggested_amount\":105500},\"sek\":{\"minimum_amount\":107000,\"suggested_amount\":199000},\"chf\":{\"minimum_amount\":9000,\"suggested_amount\":17000},\"dkk\":{\"minimum_amount\":71000,\"suggested_amount\":132500},\"nok\":{\"minimum_amount\":102500,\"suggested_amount\":190500},\"inr\":{\"minimum_amount\":1051500,\"suggested_amount\":1959500},\"jpy\":{\"minimum_amount\":17000,\"suggested_amount\":32000},\"nzd\":{\"minimum_amount\":19000,\"suggested_amount\":35500},\"pln\":{\"minimum_amount\":41500,\"suggested_amount\":76500}}},\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://read.bytesizeddesign.com\",\"hostname\":\"read.bytesizeddesign.com\",\"is_on_substack\":false,\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"pageThemes\":{\"podcast\":null},\"multiple_pins\":true,\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#7671ec\",\"primary\":{\"r\":118,\"g\":113,\"b\":236,\"a\":1},\"primary_hover\":{\"r\":98,\"g\":96,\"b\":216,\"a\":1},\"primary_elevated\":{\"r\":98,\"g\":96,\"b\":216,\"a\":1},\"secondary\":{\"r\":118,\"g\":113,\"b\":236,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":118,\"g\":113,\"b\":236,\"a\":0.2},\"bg_hover\":{\"r\":118,\"g\":113,\"b\":236,\"a\":0.3},\"dark\":{\"primary\":{\"r\":118,\"g\":113,\"b\":236,\"a\":1},\"primary_hover\":{\"r\":138,\"g\":131,\"b\":255,\"a\":1},\"primary_elevated\":{\"r\":138,\"g\":131,\"b\":255,\"a\":1},\"secondary\":{\"r\":118,\"g\":113,\"b\":236,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":118,\"g\":113,\"b\":236,\"a\":0.2},\"bg_hover\":{\"r\":118,\"g\":113,\"b\":236,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":106,\"g\":103,\"b\":224,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":118,\"g\":113,\"b\":236,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":206.90096477355226,\"g\":206.90096477355175,\"b\":206.9009647735519,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#7671ec\",\"primary\":{\"r\":118,\"g\":113,\"b\":236,\"a\":1},\"primary_hover\":{\"r\":96,\"g\":91,\"b\":233,\"a\":1},\"primary_elevated\":{\"r\":118,\"g\":113,\"b\":236,\"a\":1},\"secondary\":{\"r\":118,\"g\":113,\"b\":236,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":118,\"g\":113,\"b\":236,\"a\":1},\"primary_hover\":{\"r\":138,\"g\":131,\"b\":255,\"a\":1},\"primary_elevated\":{\"r\":138,\"g\":131,\"b\":255,\"a\":1},\"secondary\":{\"r\":118,\"g\":113,\"b\":236,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":118,\"g\":113,\"b\":236,\"a\":0.2},\"bg_hover\":{\"r\":118,\"g\":113,\"b\":236,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"secondary\":{\"r\":134,\"g\":135,\"b\":135,\"a\":1},\"tertiary\":{\"r\":146,\"g\":146,\"b\":146,\"a\":1},\"accent\":{\"r\":118,\"g\":113,\"b\":236,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":118,\"g\":113,\"b\":236,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"secondary\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"secondary_elevated\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"tertiary\":{\"r\":221,\"g\":221,\"b\":221,\"a\":1},\"quaternary\":{\"r\":183,\"g\":183,\"b\":183,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}},\"wordmark_bg\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"is_dark\":false},\"fonts\":{\"heading\":\"sans\",\"body\":\"fancy_serif\"}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[127.5,127.5,127.5],\"population\":0},\"DarkVibrant\":{\"rgb\":[66.3,66.3,66.3],\"population\":0},\"LightVibrant\":{\"rgb\":[188.7,188.7,188.7],\"population\":0},\"Muted\":{\"rgb\":[120,120,120],\"population\":78},\"DarkMuted\":{\"rgb\":[52,52,52],\"population\":5496},\"LightMuted\":{\"rgb\":[188,188,188],\"population\":17}}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":false,\"customDomain\":\"read.bytesizeddesign.com\"},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":true,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":true,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":null,\"twitterPixelId\":null,\"parselyPixelId\":null,\"googleTagManagerToken\":null,\"googleAnalytics4Token\":null,\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":false,\"credit_token_enabled\":true,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":true,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":true,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":null,\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"System design made simple for 43,000+ engineers. Weekly breakdowns of how real companies scale (GitLab, Spotify, OpenAI), post-mortems from outages you can learn from, and career advice that cuts through the noise. No fluff. No buzzwords. Just better engineering judgment.\\n\",\"notes_feed_enabled\":true,\"paywall_unlock_tokens\":true,\"post_preview_crop_gravity\":\"center\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":true,\"reader_referrals_leaderboard_enabled\":true,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":false,\"local_currency_modal_seen\":true,\"local_payment_methods_modal_seen\":true,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":true,\"welcome_page_opt_out_text\":\"Next\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":true,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"only_paid\",\"audience_before_archived\":null,\"canonical_url\":\"https://read.bytesizeddesign.com/p/how-gitlab-lost-300gb-of-production\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":156355259,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2025-02-03T06:36:13.709Z\",\"updated_at\":\"2026-08-30T18:48:22.876Z\",\"publication_id\":1499683,\"search_engine_description\":null,\"search_engine_title\":null,\"section_id\":null,\"should_send_free_preview\":true,\"show_guest_bios\":true,\"slug\":\"how-gitlab-lost-300gb-of-production\",\"social_title\":null,\"subtitle\":\"The Hard Lesson on Backups, Disaster Recovery, and Human Error\",\"teaser_post_eligible\":true,\"title\":\"How GitLab Lost 300GB of Production Data and What We Can Learn\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"only_paid\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":6,\"reactions\":{\"\u2764\":43},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"how-deepseek-v3-brings-open-source\",\"next_post_slug\":\"the-tech-leads-guide-to-load-testing\",\"cover_image\":\"https://substack-post-media.s3.amazonaws.com/public/images/0d54e327-71cc-4852-8249-5b2e7029fc30_5599x3733.jpeg\",\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":{\"post_id\":156355259,\"podcast_episode_number\":null,\"podcast_season_number\":null,\"podcast_episode_type\":null,\"should_syndicate_to_other_feed\":null,\"syndicate_to_section_id\":null,\"hide_from_feed\":false,\"free_podcast_url\":null,\"free_podcast_duration\":null,\"preview_contains_ad\":false,\"was_imported_self_serve_sync\":false,\"draft_free_podcast_url\":null,\"draft_free_podcast_duration\":null},\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"The Hard Lesson on Backups, Disaster Recovery, and Human Error\",\"body_html\":\"<p>On January 31, 2017, GitLab.com went down hard. For hours, engineers scrambled to recover, only to realize their backup system had been broken for weeks. By the time the dust settled, they had lost 5,000 projects, 700 user accounts, and countless issues and comments.</p><p>The root cause? A single mistaken command that wiped their primary database.</p><p>In a post-mortem, GitLab detailed how they lost critical production data, the mistakes that led to the disaster, and the systemic failures that made recovery nearly impossible.</p><p>Let\u2019s break it down:</p><h3>\uD83D\uDE80 <strong>TL;DR</strong></h3><p>GitLab.com suffered a major outage after an engineer accidentally <strong>deleted the primary database instead of a secondary replica.</strong> The disaster was compounded by broken backups, slow recovery tools, and a failure to catch issues before they escalated.</p><p>Here\u2019s what we\u2019ll cover:</p><p>1\uFE0F\u20E3 <strong>\uD83D\uDEE0\uFE0F What Happened?</strong> (The moment that wiped GitLab\u2019s data)<br>2\uFE0F\u20E3 <strong>\uD83D\uDED1 Root Causes</strong> (The silent failures that made recovery impossible)<br>3\uFE0F\u20E3 <strong>\uD83E\uDD14 Lessons Learned</strong> (How GitLab changed its approach)<br>4\uFE0F\u20E3 <strong>\uD83C\uDFD7\uFE0F How to Protect Your Own Systems</strong> (Paid)</p><p>\uD83D\uDCA1 <strong>More Insights & Recovery Strategies </strong>(Paid)</p><div><hr></div><h2><strong>\uD83D\uDEE0\uFE0F What Happened?</strong></h2><p>It started as a <strong>routine maintenance task.</strong></p><p>To test new database scaling techniques, a GitLab engineer needed to resynchronize a secondary PostgreSQL database. The process required wiping its data directory before restoring a fresh copy from the primary database.</p><p>\uD83D\uDEA8 <strong>The mistake:</strong> Instead of running the command on the secondary server, the engineer executed it on the <strong>primary database.</strong></p><p>\uD83D\uDCBE <strong>300GB of production data vanished in seconds.</strong></p><p>Panic set in. The team turned to their backups\u2014only to find that their <strong>backup system had been failing silently for weeks.</strong></p><p>With no working backups and a painfully slow recovery process, GitLab had no choice but to <strong>rebuild from a 6-hour-old snapshot, permanently losing all data created after that point.</strong></p><div><hr></div><h2><strong>\uD83D\uDED1 Root Causes</strong></h2><h3><strong>1\uFE0F\u20E3 No Protection Against Accidental Deletion</strong></h3><ul><li><p>There was no safeguard to prevent engineers from running destructive commands on production systems.</p></li><li><p>The primary and secondary databases <strong>looked identical</strong> in the terminal making it easy to confuse them.</p></li></ul><h3><strong>2\uFE0F\u20E3 Backups Had Been Failing Silently for Weeks</strong></h3><ul><li><p>GitLab\u2019s automated <code>pg_dump</code> backups <strong>never actually ran</strong> due to a misconfiguration.</p></li><li><p>Backup failure alerts <strong>were sent via email</strong>\u2014but those emails were silently rejected due to DMARC settings.</p></li></ul><h3><strong>3\uFE0F\u20E3 The Recovery Process Was Too Slow</strong></h3><ul><li><p>Azure disk snapshots were available but <strong>took over 18 hours to restore</strong> due to slow disk speeds.</p></li><li><p>There was no clear, <strong>battle-tested disaster recovery</strong> process to follow.</p></li></ul><h3><strong>4\uFE0F\u20E3 Replication Lag Left No Secondary Backup</strong></h3><ul><li><p>The secondary database was out of sync due to a <strong>spike in database load</strong> earlier that day.</p></li><li><p>When replication stopped, <strong>old transaction logs were purged</strong>, meaning the secondary couldn't be used as a failover.</p></li></ul><p><strong>In short: GitLab lost production data because of a mix of human error, broken backups, and an untested recovery process.</strong></p><div><hr></div><h2><strong>\uD83E\uDD14 Lessons Learned</strong></h2><p>GitLab\u2019s failure highlights <strong>five critical lessons</strong> for any engineering team:</p><h3><strong>1\uFE0F\u20E3 Your Backups Are Useless Until You Test Them</strong></h3><ul><li><p>If you don\u2019t regularly <strong>restore from backups</strong>, they might not work when you need them.</p></li><li><p><strong>GitLab assumed their backups worked.</strong> They didn't.</p></li></ul><h3><strong>2\uFE0F\u20E3 Critical Systems Need Guardrails</strong></h3><ul><li><p>Running destructive commands on production databases should require <strong>extra confirmation steps.</strong></p></li><li><p>Engineers should see <strong>clear visual indicators</strong> showing whether they\u2019re working on production or staging.</p></li></ul><h3><strong>3\uFE0F\u20E3 Recovery Speed Matters More Than Backup Frequency</strong></h3><ul><li><p>Even if backups exist, if it takes <strong>18+ hours to restore</strong>, your business will suffer.</p></li><li><p><strong>Solution:</strong> Invest in <strong>point-in-time recovery (PITR)</strong> instead of relying solely on daily snapshots.</p></li></ul><h3><strong>4\uFE0F\u20E3 Test Disaster Recovery Like You Test Your Code</strong></h3><ul><li><p>Have a <strong>documented, rehearsed plan</strong> for recovering from catastrophic failure.</p></li><li><p>Run <strong>fire drills</strong> where engineers simulate total database loss and restore from backups.</p></li><li><p>Build automated chaos tests when and where possible throughout your pipelines. <a href=\\\"https://bytesizeddesign.substack.com/p/the-engineers-guide-to-observability\\\">Read more here</a>.</p><div class=\\\"comment\\\" data-attrs=\\\"{"url":"https://open.substack.com/home","commentId":86370529,"comment":{"id":86370529,"date":"2025-01-15T18:11:14.836Z","edited_at":null,"body":"\uD83D\uDEA8 Slack learned the hard way that even the best systems can fail, but they turned their lessons into a culture of resilience.\\\\n\\\\nTheir chaos engineering approach breaks systems on purpose to discover hidden weaknesses before users feel the impact.\\\\n\\\\nFrom refining Kibana recovery to automating solutions, Slack turned failure into an engaging team sport. Chaos tests like network partitions and zone failures have made failure... boring (and that\u2019s a good thing).\\\\n\\\\nFailure is inevitable. Preparation isn\u2019t. \\\\n\\\\nRead more: https://bytesizeddesign.substack.com/p/slack-breaks-stuff-on-purpose\\\\n\\\\n\uD83D\uDCA1 How does your team handle chaos?\\\\n\\\\n\uD83D\uDE80 Learn more about building resilient systems with https://bytesizeddesign.substack.com/\\\\n\\\\n\u267B Restack to share the power of chaos testing!","body_json":{"type":"doc","attrs":{"schemaVersion":"v1"},"content":[{"type":"paragraph","content":[{"type":"text","text":"\uD83D\uDEA8 Slack learned the hard way that even the best systems can fail, but they turned their lessons into a culture of resilience."}]},{"type":"paragraph","content":[{"type":"text","text":"Their chaos engineering approach "},{"type":"text","marks":[{"type":"italic"}],"text":"breaks systems on purpose"},{"type":"text","text":" to discover hidden weaknesses before users feel the impact."}]},{"type":"paragraph","content":[{"type":"text","text":"From refining Kibana recovery to automating solutions, Slack turned failure into an engaging team sport. Chaos tests like network partitions and zone failures have made failure... boring (and that\u2019s a good thing)."}]},{"type":"paragraph","content":[{"type":"text","text":"Failure is inevitable. Preparation isn\u2019t. "}]},{"type":"paragraph","content":[{"type":"text","text":"Read more: "},{"type":"text","marks":[{"type":"link","attrs":{"href":"https://bytesizeddesign.substack.com/p/slack-breaks-stuff-on-purpose","target":"_blank","rel":"nofollow ugc noopener","class":"note-link"}}],"text":"https://bytesizeddesign.substack.com/p/slack-breaks-stuff-on-purpose"}]},{"type":"paragraph","content":[{"type":"text","text":"\uD83D\uDCA1 How does your team handle chaos?"}]},{"type":"paragraph","content":[{"type":"text","text":"\uD83D\uDE80 Learn more about building resilient systems with "},{"type":"text","marks":[{"type":"link","attrs":{"href":"https://bytesizeddesign.substack.com/","target":"_blank","rel":"nofollow ugc noopener","class":"note-link"}}],"text":"https://bytesizeddesign.substack.com/"}]},{"type":"paragraph","content":[{"type":"text","text":"\u267B Restack to share the power of chaos testing!"}]}]},"restacks":2,"reaction_count":9,"attachments":[],"name":"Byte-Sized Design","user_id":104689928,"photo_url":"https://substack-post-media.s3.amazonaws.com/public/images/75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png","user_bestseller_tier":100},"source":null,"forumChannel":null}\\\" data-component-name=\\\"CommentPlaceholder\\\"></div></li></ul><h3><strong>5\uFE0F\u20E3 Redundancy Must Be Truly Redundant</strong></h3><ul><li><p>A secondary database that <strong>isn\u2019t truly in sync</strong> won\u2019t save you in a crisis.</p></li><li><p>GitLab fixed this by adding <strong>multiple secondaries across different regions.</strong></p></li></ul><div><hr></div><h2><strong>\uD83C\uDFD7\uFE0F How to Protect Your Own Systems</strong></h2><p>Many companies have similar risks lurking in their infrastructure. Here\u2019s what you can do to avoid disaster:</p>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"On January 31, 2017, GitLab.com went down hard. For hours, engineers scrambled to recover, only to realize their backup system had been broken for weeks. By the time the dust settled, they had lost 5,000 projects, 700 user accounts, and countless issues and comments.\",\"wordcount\":1114,\"post_preview_limit\":200,\"language\":\"en\",\"postTags\":[{\"id\":\"2a228b2d-3525-429e-94f0-c138d4a9054f\",\"publication_id\":1499683,\"name\":\"disaster recovery\",\"slug\":\"disaster-recovery\",\"hidden\":false},{\"id\":\"3f4ccac8-cd11-4974-a504-76d88bb5e8a4\",\"publication_id\":1499683,\"name\":\"programming\",\"slug\":\"programming\",\"hidden\":false},{\"id\":\"636c6117-53ae-4f91-8f12-a66ebda7f0db\",\"publication_id\":1499683,\"name\":\"post-mortem\",\"slug\":\"post-mortem\",\"hidden\":false},{\"id\":\"80053665-d2f7-4d78-aae2-20886ecd6765\",\"publication_id\":1499683,\"name\":\"database\",\"slug\":\"database\",\"hidden\":false},{\"id\":\"835da990-fd2c-4725-8dad-12919476eaf4\",\"publication_id\":1499683,\"name\":\"system design\",\"slug\":\"system-design\",\"hidden\":false},{\"id\":\"f959d5d9-313e-4d1a-83ef-ff8914c9f587\",\"publication_id\":1499683,\"name\":\"availability\",\"slug\":\"availability\",\"hidden\":false}],\"postCountryBlocks\":[],\"headlineTest\":null,\"coverImagePalette\":{\"Vibrant\":{\"rgb\":[243,130,4],\"population\":91},\"DarkVibrant\":{\"rgb\":[143,17,23],\"population\":66},\"LightVibrant\":{\"rgb\":[227,171,117],\"population\":10},\"Muted\":{\"rgb\":[148,84,92],\"population\":1},\"DarkMuted\":{\"rgb\":[42,68,78],\"population\":101},\"LightMuted\":{\"rgb\":[180,188,189],\"population\":672}},\"publishedBylines\":[{\"id\":104689928,\"name\":\"Byte-Sized Design\",\"handle\":\"bytesizeddesign\",\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/75986514-8b28-4199-95ab-d22806dd8fe1_228x228.png\",\"bio\":\"Real talk about building systems that work. Career advice for growing engineers, how actual companies solve large-scale problems, post-mortems from things that broke, and the system design fundamentals that matter. No fluff, no buzzwords.\",\"profile_set_up_at\":\"2024-11-01T03:15:52.544Z\",\"reader_installed_at\":\"2025-04-04T11:03:33.669Z\",\"publicationUsers\":[{\"id\":1466741,\"user_id\":104689928,\"publication_id\":1499683,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":1499683,\"name\":\"Byte-Sized Design\",\"subdomain\":\"bytesizeddesign\",\"custom_domain\":\"read.bytesizeddesign.com\",\"custom_domain_optional\":false,\"hero_text\":\"Master system design concepts, engineering fundamentals, and interview basics. Weekly summaries, post-mortems, and advice for 43,000+ engineers.\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/06b64927-5de1-4edc-a245-b9b486e07503_1024x1024.png\",\"author_id\":104689928,\"primary_user_id\":104689928,\"theme_var_background_pop\":\"#BAA049\",\"created_at\":\"2023-03-16T21:02:31.841Z\",\"email_from_name\":\"Byte-Sized Design\",\"copyright\":\"Byte-Sized Design\",\"founding_plan_name\":\"Founding Member\",\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"enabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"magaziney\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"is_guest\":false,\"bestseller_tier\":100,\"status\":{\"bestsellerTier\":100,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"bestseller\",\"tier\":100},\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":43,\"comment_count\":0,\"child_comment_count\":0,\"audio_items\":[{\"post_id\":156355259,\"voice_id\":\"en-US-AlloyTurboMultilingualNeural\",\"audio_url\":null,\"type\":\"tts\",\"status\":\"paywalled\"}],\"is_geoblocked\":false,\"hidden\":true,\"hasCashtag\":false},\"comments\":[],\"canonicalUrl\":\"https://read.bytesizeddesign.com/p/how-gitlab-lost-300gb-of-production\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://read.bytesizeddesign.com/p/how-gitlab-lost-300gb-of-production\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":\"#7671ec\",\"background_pop\":\"#7671ec\",\"color_theme_bg_web\":\"#ffffff\",\"cover_bg_color\":\"#ffffff\",\"cover_bg_color_secondary\":\"#f0f0f0\",\"background_pop_darken\":\"#605be9\",\"print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_darken\":\"#605be9\",\"color_theme_print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_20\":\"rgba(118, 113, 236, 0.2)\",\"color_theme_bg_pop_30\":\"rgba(118, 113, 236, 0.3)\",\"print_pop\":\"#7671ec\",\"color_theme_accent\":\"#7671ec\",\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#7671ec\",\"font_family_headings_preset\":\"'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'\",\"font_weight_headings_preset\":700,\"font_family_body_preset\":\"Lora,sans-serif\",\"font_weight_body_preset\":400,\"font_size_body_offset\":\"1px\",\"font_preset_heading\":\"sans\",\"font_preset_body\":\"fancy_serif\",\"home_hero\":\"magazine-5\",\"home_posts\":\"list\",\"home_show_top_posts\":true,\"web_bg_color\":\"#ffffff\",\"background_contrast_1\":\"#f0f0f0\",\"background_contrast_2\":\"#dddddd\",\"background_contrast_3\":\"#b7b7b7\",\"background_contrast_4\":\"#929292\",\"background_contrast_5\":\"#515151\",\"color_theme_bg_contrast_1\":\"#f0f0f0\",\"color_theme_bg_contrast_2\":\"#dddddd\",\"color_theme_bg_contrast_3\":\"#b7b7b7\",\"color_theme_bg_contrast_4\":\"#929292\",\"color_theme_bg_contrast_5\":\"#515151\",\"color_theme_bg_elevated\":\"#ffffff\",\"color_theme_bg_elevated_secondary\":\"#f0f0f0\",\"color_theme_bg_elevated_tertiary\":\"#dddddd\",\"color_theme_detail\":\"#e6e6e6\",\"background_contrast_pop\":\"rgba(118, 113, 236, 0.4)\",\"color_theme_bg_contrast_pop\":\"rgba(118, 113, 236, 0.4)\",\"theme_bg_is_dark\":\"0\",\"print_on_web_bg_color\":\"#363737\",\"print_secondary_on_web_bg_color\":\"#868787\",\"background_pop_rgb\":\"118, 113, 236\",\"color_theme_bg_pop_rgb\":\"118, 113, 236\",\"color_theme_accent_rgb\":\"118, 113, 236\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":{\"id\":\"70089b2a\",\"trial_period_days\":7},\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"125.0\",\"major\":\"125\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
|
||
<script nonce="xBzf9d9OeqN6zCc0ysemKQ">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"bytesizeddesign\",\"publication_id\":1499683,\"has_plans\":true,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
|
||
|
||
|
||
|
||
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.7371871e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.571db0c6.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.a0731d37.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.d196682b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.2686bb5d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.ae8857e4.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.4614ce58.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.41564edd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.b2140fcd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.f3dc2b43.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.7aa24e85.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.1820773d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.765aca5a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.2f1bdde5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.4e726ad5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.f03f24de.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.f579d62b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.80c75c78.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.92b9fc9c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.258e49f2.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.d493457b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.ce61e623.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.015b0f71.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.f1207089.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.b91ab2ef.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.b93f6845.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89356.66161d1e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.211a7337.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47925.24a01d36.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.03bd4c41.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71505.3dc43e33.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.5d148f2c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.86d95e8c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.a5a682c3.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/86118.ff1ec843.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.7fe128ac.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.c78aabc1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.d6e9f8ec.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/29231.a7af5911.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.2b32bbb6.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.f1c17ded.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.1d0af2b9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.951c11cb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.58e078b0.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.dfde36c8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.94c6b3f1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.a327e9f4.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.8bb0d59c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.8f5c5991.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.cb326b80.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.680833b5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.df6a856d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.cf3bcc2c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.8a1bc5dd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.3f20fb36.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.bdbc4e21.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58483.3adddb8a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.0a97506f.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.09a9082f.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.7f0face7.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/92610.a4fc1daf.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.cb10b4b8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11244.b723f409.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99260.28b0ed6a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.7d5c5ba3.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/76768.9a2f2bae.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/41932.6c688f22.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12114.979fb38e.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.9732c6f9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.4f5ae3be.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53871.2923c5c9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.588a06d1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99068.bd2e1fba.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.79d3f415.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.c7f428fb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.098bcbcf.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.2eb51db5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.72d1d844.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.bbf1c283.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.2e8ed950.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/32552.889dd584.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.c287f2d8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.6c265246.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.9b9761ae.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.b40bf629.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.07cd629b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.35b48b09.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.f37fb930.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.24075434.js" charset="utf-8"></script>
|
||
|
||
|
||
<script nonce="xBzf9d9OeqN6zCc0ysemKQ" nomodule>
|
||
(function() {
|
||
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
|
||
var warningDiv = document.createElement('div');
|
||
warningDiv.style.color = 'red';
|
||
warningDiv.style.padding = '10px';
|
||
warningDiv.style.margin = '10px 0';
|
||
warningDiv.style.border = '1px solid red';
|
||
warningDiv.style.backgroundColor = 'lightyellow';
|
||
warningDiv.innerText = message;
|
||
document.body.prepend(warningDiv);
|
||
})();
|
||
</script>
|
||
|
||
|
||
<!-- Datadog Analytics -->
|
||
<script nonce="xBzf9d9OeqN6zCc0ysemKQ">
|
||
(function(h,o,u,n,d) {
|
||
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
|
||
d=o.createElement(u);d.async=1;d.src=n
|
||
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
|
||
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
|
||
window.DD_RUM.onReady(function() {
|
||
window.DD_RUM.init({
|
||
clientToken: 'puba71073f072643721169b68f352438710',
|
||
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
|
||
site: 'datadoghq.com',
|
||
service: 'web',
|
||
env: window._preloads.dd_env,
|
||
version: '8b4c00716b2bc0f7ca1fded553014f7e37b5b07a',
|
||
sessionSampleRate: 1,
|
||
sessionReplaySampleRate: 10,
|
||
trackUserInteractions: window._preloads.dd_ti,
|
||
trackResources: true,
|
||
trackLongTasks: true,
|
||
defaultPrivacyLevel: 'mask-user-input',
|
||
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
|
||
});
|
||
})
|
||
</script>
|
||
<!-- End Datadog Analytics -->
|
||
|
||
<!-- Cloudflare Web Analytics -->
|
||
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
|
||
<!-- End Cloudflare Web Analytics -->
|
||
|
||
|
||
<!-- Fallback tracking pixels -->
|
||
|
||
|
||
|
||
|
||
<noscript>
|
||
<style>
|
||
#nojs-banner {
|
||
position: fixed;
|
||
bottom: 0;
|
||
left: 0;
|
||
padding: 16px 16px 16px 32px;
|
||
width: 100%;
|
||
box-sizing: border-box;
|
||
background: red;
|
||
color: white;
|
||
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
|
||
font-size: 13px;
|
||
line-height: 13px;
|
||
}
|
||
#nojs-banner a {
|
||
color: inherit;
|
||
text-decoration: underline;
|
||
}
|
||
</style>
|
||
|
||
<div id="nojs-banner">
|
||
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
|
||
</div>
|
||
</noscript>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="xBzf9d9OeqN6zCc0ysemKQ" data-cf-beacon='{"version":"2024.11.0","token":"68cfe66b5c4749e2ba64d4d9640c04c0","spa":2}' crossorigin="anonymous"></script>
|
||
</body>
|
||
</html>
|